labs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thors...@apache.org
Subject svn commit: r679773 - in /labs/droids/trunk: ./ src/core/java/org/apache/droids/ src/core/java/org/apache/droids/api/ src/core/java/org/apache/droids/delay/
Date Fri, 25 Jul 2008 12:08:40 GMT
Author: thorsten
Date: Fri Jul 25 05:08:40 2008
New Revision: 679773

URL: http://svn.apache.org/viewvc?rev=679773&view=rev
Log:
LABS-139 Added delay between http requests.
Now you can define the milliseconds of delay between http request. You have now three different
implementations for the interface DelayTimer.
* SimpleDelayTimer
* RandomDelayTimer
* GaussianRandomDelayTimer

-- SimpleDelayTimer --
Only applies a fixed delay to the request. Set the property delay in the spring context.

-- RandomDelayTimer --
Generate a random delay between 0ms and the value specified. Set the property delay in the
spring context.

-- GaussianRandomDelayTimer --
Generate a random delay and combine with a fixed delay. Set the property delay and fixedDelay
in the spring context.

Feature submitted by Javier Puerto. Muchas gracias, monstro.

Added:
    labs/droids/trunk/src/core/java/org/apache/droids/api/DelayTimer.java   (with props)
    labs/droids/trunk/src/core/java/org/apache/droids/api/DelayWorker.java   (with props)
    labs/droids/trunk/src/core/java/org/apache/droids/delay/
    labs/droids/trunk/src/core/java/org/apache/droids/delay/GaussianRandomDelayTimer.java
  (with props)
    labs/droids/trunk/src/core/java/org/apache/droids/delay/RandomDelayTimer.java   (with
props)
    labs/droids/trunk/src/core/java/org/apache/droids/delay/SimpleDelayTimer.java   (with
props)
    labs/droids/trunk/src/core/java/org/apache/droids/delay/package-info.java   (with props)
Modified:
    labs/droids/trunk/default.properties
    labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java
    labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java
    labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
    labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml

Modified: labs/droids/trunk/default.properties
URL: http://svn.apache.org/viewvc/labs/droids/trunk/default.properties?rev=679773&r1=679772&r2=679773&view=diff
==============================================================================
--- labs/droids/trunk/default.properties (original)
+++ labs/droids/trunk/default.properties Fri Jul 25 05:08:40 2008
@@ -31,6 +31,9 @@
 # protocol handling
 droids.protocol.http.force=false
 
+#Delay between request.
+droids.delay.request=500
+
 # Threading 
 droids.maxThreads=5
 

Modified: labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java?rev=679773&r1=679772&r2=679773&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/AbstractDroid.java Fri Jul 25 05:08:40
2008
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.droids;
 
 import java.util.concurrent.ConcurrentHashMap;

Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java?rev=679773&r1=679772&r2=679773&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultCrawler.java Fri Jul 25 05:08:40
2008
@@ -23,6 +23,8 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.droids.api.DelayTimer;
+import org.apache.droids.api.DelayWorker;
 import org.apache.droids.api.Task;
 import org.apache.droids.api.Worker;
 import org.apache.droids.exception.DroidsException;
@@ -114,14 +116,28 @@
     this.url = url;
   }
 
+  private DelayTimer delayTimer;
+  
+  /**
+   * Get the DelayTimer implementation that we want to use.
+   * @return
+   */
+  public DelayTimer getDelayTimer() {
+	  return delayTimer;
+  }
+
+  public void setDelayTimer(DelayTimer delayTimer) throws DroidsException {
+	  this.delayTimer = delayTimer;
+  }
   
   /* 
    * @see org.apache.droids.AbstractDroid#getWorker()
    */
   public  Worker getWorker() throws DroidsException {
-    Worker worker = new DefaultWorker();
+    DelayWorker worker = new DefaultWorker();
       worker.setQueue(getQueue());
       worker.setDroid(this);
+      worker.setDelayTimer(getDelayTimer());
       return worker;
     
   }

Modified: labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java?rev=679773&r1=679772&r2=679773&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/DefaultWorker.java Fri Jul 25 05:08:40
2008
@@ -7,13 +7,14 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.droids.api.DelayTimer;
 import org.apache.droids.api.Droid;
 import org.apache.droids.api.Parse;
 import org.apache.droids.api.Parser;
 import org.apache.droids.api.Protocol;
 import org.apache.droids.api.Queue;
 import org.apache.droids.api.Task;
-import org.apache.droids.api.Worker;
+import org.apache.droids.api.DelayWorker;
 import org.apache.droids.exception.DroidsException;
 import org.apache.droids.helper.factories.HandlerFactory;
 import org.apache.droids.helper.factories.ParserFactory;
@@ -21,7 +22,7 @@
 import org.apache.droids.helper.factories.URLFiltersFactory;
 import org.apache.droids.parse.Outlink;
 
-public class DefaultWorker extends Thread implements Worker {
+public class DefaultWorker extends Thread implements DelayWorker {
 
   protected final Log log = LogFactory.getLog(this.getClass().getCanonicalName());
 
@@ -46,6 +47,8 @@
   private URLFiltersFactory filtersFactory;
 
   private HandlerFactory handlerFactory;
+  
+  private DelayTimer delayTimer;
 
   private int depth;
 
@@ -53,6 +56,11 @@
     final String userAgent = this.getClass().getCanonicalName();
     Core.threadMessage("Starting " + userAgent);
     try {
+      if(delayTimer.getDelayMillis() > 0)
+      {
+    	log.debug("Delay " + delayTimer.getDelayMillis() + "ms");
+        sleep(delayTimer.getDelayMillis());
+      }
       setUri(link.getId());
       Core.threadMessage("uri " + getUri());
       setProtocol(protocolFactory.getProtocol(getUri()));
@@ -205,4 +213,11 @@
     }
   }
 
+  public DelayTimer getDelayTimer() {
+	  return delayTimer;
+  }
+
+  public void setDelayTimer(DelayTimer delayTimer) throws DroidsException {
+	  this.delayTimer = delayTimer;
+  }
 }
\ No newline at end of file

Added: labs/droids/trunk/src/core/java/org/apache/droids/api/DelayTimer.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/api/DelayTimer.java?rev=679773&view=auto
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/api/DelayTimer.java (added)
+++ labs/droids/trunk/src/core/java/org/apache/droids/api/DelayTimer.java Fri Jul 25 05:08:40
2008
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.api;
+
+/**
+ * Define the timer delay interface. 
+ * @author Javier Puerto
+ *
+ */
+public interface DelayTimer {
+
+	/**
+	 * Returns the value of the delay between request.
+	 * @return
+	 */
+	public long getDelayMillis();
+}

Propchange: labs/droids/trunk/src/core/java/org/apache/droids/api/DelayTimer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: labs/droids/trunk/src/core/java/org/apache/droids/api/DelayWorker.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/api/DelayWorker.java?rev=679773&view=auto
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/api/DelayWorker.java (added)
+++ labs/droids/trunk/src/core/java/org/apache/droids/api/DelayWorker.java Fri Jul 25 05:08:40
2008
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.api;
+
+import org.apache.droids.exception.DroidsException;
+
+public interface DelayWorker extends Worker {
+
+	  /**
+	   * @param delayTimer
+	   * @throws DroidsException
+	   */
+	  void setDelayTimer(DelayTimer delayTimer) throws DroidsException;
+}

Propchange: labs/droids/trunk/src/core/java/org/apache/droids/api/DelayWorker.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: labs/droids/trunk/src/core/java/org/apache/droids/delay/GaussianRandomDelayTimer.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/delay/GaussianRandomDelayTimer.java?rev=679773&view=auto
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/delay/GaussianRandomDelayTimer.java
(added)
+++ labs/droids/trunk/src/core/java/org/apache/droids/delay/GaussianRandomDelayTimer.java
Fri Jul 25 05:08:40 2008
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.delay;
+
+import org.apache.droids.api.DelayTimer;
+
+public class GaussianRandomDelayTimer extends RandomDelayTimer implements
+		DelayTimer {
+  private long fixedDelay;
+
+  public GaussianRandomDelayTimer(){
+	  super();
+	  fixedDelay = 0;
+  }
+  
+  @Override
+  public long getDelayMillis() {
+	return (long) Math.abs((this.random.nextGaussian() * getDelay()) + fixedDelay);
+  }
+
+public long getFixedDelay() {
+	  return fixedDelay;
+  }
+
+  public void setFixedDelay(long fixedDelay) {
+	  this.fixedDelay = fixedDelay;
+  }
+}

Propchange: labs/droids/trunk/src/core/java/org/apache/droids/delay/GaussianRandomDelayTimer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: labs/droids/trunk/src/core/java/org/apache/droids/delay/RandomDelayTimer.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/delay/RandomDelayTimer.java?rev=679773&view=auto
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/delay/RandomDelayTimer.java (added)
+++ labs/droids/trunk/src/core/java/org/apache/droids/delay/RandomDelayTimer.java Fri Jul
25 05:08:40 2008
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.delay;
+
+import java.util.Random;
+
+import org.apache.droids.api.DelayTimer;
+
+public class RandomDelayTimer extends SimpleDelayTimer implements DelayTimer {
+
+	protected Random random;
+	
+	public RandomDelayTimer()
+	{
+		this.random = new Random();
+	}
+
+	@Override
+	public long getDelayMillis() {
+		return (long) Math.abs(this.random.nextDouble() * getDelay());
+	}
+	
+	
+}

Propchange: labs/droids/trunk/src/core/java/org/apache/droids/delay/RandomDelayTimer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: labs/droids/trunk/src/core/java/org/apache/droids/delay/SimpleDelayTimer.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/delay/SimpleDelayTimer.java?rev=679773&view=auto
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/delay/SimpleDelayTimer.java (added)
+++ labs/droids/trunk/src/core/java/org/apache/droids/delay/SimpleDelayTimer.java Fri Jul
25 05:08:40 2008
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.droids.delay;
+
+import org.apache.droids.api.DelayTimer;
+
+public class SimpleDelayTimer implements DelayTimer {
+
+	/**
+	 * The fixed value of the delay.
+	 */
+	private long delay = 0;
+	
+	public long getDelayMillis() {
+		return delay;
+	}
+
+	/**
+	 * Returns the delay time. 
+	 * @return long
+	 */
+	public long getDelay() {
+		return delay;
+	}
+
+	/**
+	 * Sets the delay time.
+	 * @param delay long
+	 */
+	public void setDelay(long delay) {
+		this.delay = delay;
+	}
+
+}

Propchange: labs/droids/trunk/src/core/java/org/apache/droids/delay/SimpleDelayTimer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: labs/droids/trunk/src/core/java/org/apache/droids/delay/package-info.java
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/delay/package-info.java?rev=679773&view=auto
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/delay/package-info.java (added)
+++ labs/droids/trunk/src/core/java/org/apache/droids/delay/package-info.java Fri Jul 25 05:08:40
2008
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package is the principal package for Apache Droids Delay Timers. Classes in the
+ * package are examples of various delay implementations.
+ */
+package org.apache.droids.delay;
+

Propchange: labs/droids/trunk/src/core/java/org/apache/droids/delay/package-info.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml
URL: http://svn.apache.org/viewvc/labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml?rev=679773&r1=679772&r2=679773&view=diff
==============================================================================
--- labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml (original)
+++ labs/droids/trunk/src/core/java/org/apache/droids/droids-core-context.xml Fri Jul 25 05:08:40
2008
@@ -24,6 +24,7 @@
     class="org.apache.droids.DefaultCrawler">
     <property name="core" ref="org.apache.droids.Core"/>
     <property name="queue" ref="org.apache.droids.queue.Simple"/>
+    <property name="delayTimer" ref="org.apache.droids.delay.SimpleDelayTimer"/>
     <property name="maxThreads" value="@droids.maxThreads@"/>
     <property name="url" value="@droids.initial.url@"/>
   </bean>
@@ -36,7 +37,7 @@
   <!-- Protocol -->
   <bean 
     name="org.apache.droids.api.Protocol/http"
-    class="org.apache.droids.protocol.http.Http">
+    class="org.apache.droids.protocol.http.Http" scope="prototype">
     <property name="from" value="labs@labs.apache.org"/>
     <property name="refer" value="http://svn.apache.org/repos/asf/labs/droids"/>
     <property name="userAgent" value="DROIDS-crawler-x-m01y08"/>
@@ -63,4 +64,10 @@
     <property name="outputDir" value="@droids.handler.save.dir@"/>
     <property name="includeHost" value="@droids.handler.save.includeHost@"/>
   </bean>
+  
+  <bean 
+    name="org.apache.droids.delay.SimpleDelayTimer"
+    class="org.apache.droids.delay.SimpleDelayTimer">
+    <property name="delay" value="@droids.delay.request@"/>
+  </bean>
 </beans>



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org


Mime
View raw message