streams-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mfrank...@apache.org
Subject [6/8] git commit: STREAMS-144 | Added max batch size to AbstractInstagramProvider
Date Mon, 25 Aug 2014 16:42:53 GMT
STREAMS-144 | Added max batch size to AbstractInstagramProvider


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/96bdf954
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/96bdf954
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/96bdf954

Branch: refs/heads/master
Commit: 96bdf954588da0edfe42fba3df82bddb82b98aaa
Parents: 2d0e63c
Author: Ryan Ebanks <rebanks@Informations-MacBook-Pro-3.local>
Authored: Tue Aug 19 09:46:53 2014 -0500
Committer: Ryan Ebanks <rebanks@Informations-MacBook-Pro-3.local>
Committed: Tue Aug 19 09:46:53 2014 -0500

----------------------------------------------------------------------
 .../streams/instagram/provider/InstagramAbstractProvider.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/96bdf954/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramAbstractProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramAbstractProvider.java
b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramAbstractProvider.java
index e08dc04..3f12493 100644
--- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramAbstractProvider.java
+++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramAbstractProvider.java
@@ -43,6 +43,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
  */
 public abstract class InstagramAbstractProvider implements StreamsProvider {
 
+    private static final int MAX_BATCH_SIZE = 2000;
+
     protected InstagramConfiguration config;
     private InstagramDataCollector dataCollector;
     protected Queue<StreamsDatum> dataQueue; //exposed for testing
@@ -74,8 +76,10 @@ public abstract class InstagramAbstractProvider implements StreamsProvider
{
     @Override
     public StreamsResultSet readCurrent() {
         Queue<StreamsDatum> batch = Queues.newConcurrentLinkedQueue();
-        while(!this.dataQueue.isEmpty()) {
+        int count = 0;
+        while(!this.dataQueue.isEmpty() && count < MAX_BATCH_SIZE) {
             ComponentUtils.offerUntilSuccess(ComponentUtils.pollWhileNotEmpty(this.dataQueue),
batch);
+            ++count;
         }
         this.isCompleted.set(batch.size() == 0 && this.dataQueue.isEmpty() &&
this.dataCollector.isCompleted());
         return new StreamsResultSet(batch);


Mime
View raw message