streams-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sblack...@apache.org
Subject [2/2] incubator-streams git commit: apply max items to following provider
Date Fri, 23 Oct 2015 20:52:50 GMT
apply max items to following provider


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/8154b040
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/8154b040
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/8154b040

Branch: refs/heads/master
Commit: 8154b0406d5645db583f3e3271293ab146a26510
Parents: 8d89f4b
Author: Steve Blackmon <sblackmon@apache.org>
Authored: Tue Oct 20 13:06:06 2015 -0500
Committer: Steve Blackmon <sblackmon@apache.org>
Committed: Tue Oct 20 13:06:06 2015 -0500

----------------------------------------------------------------------
 .../provider/TwitterFollowingProviderTask.java        | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8154b040/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProviderTask.java
b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProviderTask.java
index e96dbf1..e298827 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProviderTask.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProviderTask.java
@@ -49,6 +49,7 @@ public class TwitterFollowingProviderTask implements Runnable {
     protected String endpoint;
 
     private int max_per_page = 200;
+    int count = 0;
 
     public TwitterFollowingProviderTask(TwitterFollowingProvider provider, Twitter twitter,
Long id, String endpoint, Boolean idsOnly) {
         this.provider = provider;
@@ -134,10 +135,13 @@ public class TwitterFollowingProviderTask implements Runnable {
                                     .withFollowee(mapper.readValue(otherJson, User.class))
                                     .withFollower(mapper.readValue(userJson, User.class));
                         }
-                        ComponentUtils.offerUntilSuccess(new StreamsDatum(follow), provider.providerQueue);
 
                         Preconditions.checkNotNull(follow);
 
+                        if( count < provider.getConfig().getMaxItems()) {
+                            provider.addDatum(new StreamsDatum(follow));
+                            count++;
+                        }
 
                     } catch (Exception e) {
                         LOGGER.warn("Exception: {}", e);
@@ -153,7 +157,7 @@ public class TwitterFollowingProviderTask implements Runnable {
             catch(Exception e) {
                 keepTrying += TwitterErrorHandler.handleTwitterError(client, e);
             }
-        } while (curser != 0 && keepTrying < 10);
+        } while (curser != 0 && keepTrying < 10 && count < provider.getConfig().getMaxItems());
     }
 
     private void collectIds(Long id) {
@@ -191,6 +195,10 @@ public class TwitterFollowingProviderTask implements Runnable {
 
                         Preconditions.checkNotNull(follow);
 
+                        if( count < provider.getConfig().getMaxItems()) {
+                            provider.addDatum(new StreamsDatum(follow));
+                            count++;
+                        }
                     } catch (Exception e) {
                         LOGGER.warn("Exception: {}", e);
                     }
@@ -205,7 +213,7 @@ public class TwitterFollowingProviderTask implements Runnable {
             catch(Exception e) {
                 keepTrying += TwitterErrorHandler.handleTwitterError(client, e);
             }
-        } while (curser != 0 && keepTrying < 10);
+        } while (curser != 0 && keepTrying < 10 && count < provider.getConfig().getMaxItems());
     }
 
     protected void getFollowing(String screenName) {


Mime
View raw message