camel-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ningji...@apache.org
Subject svn commit: r1441326 - in /camel/trunk/components/camel-twitter/src: main/java/org/apache/camel/component/twitter/ main/java/org/apache/camel/component/twitter/consumer/search/ main/java/org/apache/camel/component/twitter/producer/ test/java/org/apache...
Date Fri, 01 Feb 2013 07:42:13 GMT
Author: ningjiang
Date: Fri Feb  1 07:42:13 2013
New Revision: 1441326

URL: http://svn.apache.org/viewvc?rev=1441326&view=rev
Log:
CAMEL-6021,CAMEL-6022 added the option of count and numberOfPages with thanks to Luca

Modified:
    camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java
    camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java
    camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java
    camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java
    camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java

Modified: camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java
(original)
+++ camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConfiguration.java
Fri Feb  1 07:42:13 2013
@@ -86,6 +86,11 @@ public class TwitterConfiguration {
      * Used ot set the preferred language on which to search
      */
     private String lang;
+
+    /**
+     * Used to set the maximum tweets per page (max = 100)
+     */
+    private Integer count;
     
     private Date parsedDate;
     private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
@@ -101,6 +106,11 @@ public class TwitterConfiguration {
     private TwitterStream twitterStream;
 
     /**
+     * Number of page to iterate before stop (default is 1)
+     */
+    private Integer numberOfPages = new Integer(1);
+
+    /**
      * Ensures required fields are available.
      */
     public void checkComplete() {
@@ -271,6 +281,22 @@ public class TwitterConfiguration {
     public void setLang(String lang) {
         this.lang = lang;
     }
+
+    public Integer getCount() {
+        return count;
+    }
+
+    public void setCount(int count) {
+        this.count = count;
+    }
+
+    public Integer getNumberOfPages() {
+        return numberOfPages;
+    }
+
+    public void setNumberOfPages(Integer numberOfPages) {
+        this.numberOfPages = numberOfPages;
+    }
 }
 
 

Modified: camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java
(original)
+++ camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/TwitterConstants.java
Fri Feb  1 07:42:13 2013
@@ -24,6 +24,8 @@ public final class TwitterConstants {
     public static final String TWITTER_KEYWORDS = "CamelTwitterKeywords";
 
     public static final String TWITTER_SEARCH_LANGUAGE = "CamelTwitterSearchLanguage";
+    public static final String TWITTER_COUNT = "CamelTwitterCount";
+    public static final String TWITTER_NUMBER_OF_PAGES = "CamelTwitterNumberOfPages";
 
     private TwitterConstants() {
         // utility

Modified: camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java
(original)
+++ camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/consumer/search/SearchConsumer.java
Fri Feb  1 07:42:13 2013
@@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
 import twitter4j.Query;
 import twitter4j.QueryResult;
 import twitter4j.Status;
+import twitter4j.Twitter;
 import twitter4j.TwitterException;
 
 /**
@@ -41,35 +42,59 @@ public class SearchConsumer extends Twit
     }
 
     public List<Status> pollConsume() throws TwitterException {
+        
         String keywords = te.getProperties().getKeywords();
         Query query = new Query(keywords);
         if (te.getProperties().isFilterOld()) {
             query.setSinceId(lastId);
         }
-        if (ObjectHelper.isNotEmpty(te.getProperties().getLang())) {
-            query.setLang(te.getProperties().getLang());
-        }
+        
         LOG.debug("Searching twitter with keywords: {}", keywords);
         return search(query);
     }
 
     public List<Status> directConsume() throws TwitterException {
+        
         String keywords = te.getProperties().getKeywords();
         if (keywords == null || keywords.trim().length() == 0) {
             return Collections.emptyList();
         }
         Query query = new Query(keywords);
-        if (ObjectHelper.isNotEmpty(te.getProperties().getLang())) {
-            query.setLang(te.getProperties().getLang());
-        }
+        
         LOG.debug("Searching twitter with keywords: {}", keywords);
         return search(query);
     }
 
     private List<Status> search(Query query) throws TwitterException {
-        QueryResult qr = te.getProperties().getTwitter().search(query);
+        Integer numberOfPages = 1;
+        
+        if (ObjectHelper.isNotEmpty(te.getProperties().getLang())) {
+            query.setLang(te.getProperties().getLang());
+        }
+
+        if (ObjectHelper.isNotEmpty(te.getProperties().getCount())) {
+            query.setCount(te.getProperties().getCount());
+        }
+
+        if (ObjectHelper.isNotEmpty(te.getProperties().getNumberOfPages())) {
+            numberOfPages = te.getProperties().getNumberOfPages();
+        }
+        
+        LOG.debug("Searching with " + numberOfPages + " pages.");
+
+        Twitter twitter = te.getProperties().getTwitter();
+        QueryResult qr = twitter.search(query);
         List<Status> tweets = qr.getTweets();
 
+        for (int i = 1; i < numberOfPages; i++) {
+            if (!qr.hasNext()) {
+                break;
+            }
+
+            qr = twitter.search(qr.nextQuery());
+            tweets.addAll(qr.getTweets());
+        }
+
         if (te.getProperties().isFilterOld()) {
             for (Status t : tweets) {
                 checkLastId(t.getId());

Modified: camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java
(original)
+++ camel/trunk/components/camel-twitter/src/main/java/org/apache/camel/component/twitter/producer/SearchProducer.java
Fri Feb  1 07:42:13 2013
@@ -16,8 +16,6 @@
  */
 package org.apache.camel.component.twitter.producer;
 
-import java.util.List;
-
 import org.apache.camel.CamelExchangeException;
 import org.apache.camel.Exchange;
 import org.apache.camel.component.twitter.TwitterConstants;
@@ -28,6 +26,8 @@ import twitter4j.QueryResult;
 import twitter4j.Status;
 import twitter4j.Twitter;
 
+import java.util.List;
+
 public class SearchProducer extends Twitter4JProducer {
 
     private volatile long lastId;
@@ -39,6 +39,7 @@ public class SearchProducer extends Twit
     @Override
     public void process(Exchange exchange) throws Exception {
         long myLastId = lastId;
+        // KEYWORDS
         // keywords from header take precedence
         String keywords = exchange.getIn().getHeader(TwitterConstants.TWITTER_KEYWORDS, String.class);
         if (keywords == null) {
@@ -48,12 +49,15 @@ public class SearchProducer extends Twit
         if (keywords == null) {
             throw new CamelExchangeException("No keywords to use for query", exchange);
         }
-        
+
         Query query = new Query(keywords);
+
+        // filter of older tweets
         if (te.getProperties().isFilterOld() && myLastId != 0) {
             query.setSinceId(myLastId);
         }
-        
+
+        // language
         String lang = exchange.getIn().getHeader(TwitterConstants.TWITTER_SEARCH_LANGUAGE,
String.class);
         if (lang == null) {
             lang = te.getProperties().getLang();
@@ -63,11 +67,35 @@ public class SearchProducer extends Twit
             query.setLang(lang);
         }
 
+        // number of elemnt per page
+        Integer count = exchange.getIn().getHeader(TwitterConstants.TWITTER_COUNT, Integer.class);
+        if (count == null) {
+            count = te.getProperties().getCount();
+        }
+        if (ObjectHelper.isNotEmpty(count)) {
+            query.setCount(count);
+        }
+
+        // number of pages
+        Integer numberOfPages = exchange.getIn().getHeader(TwitterConstants.TWITTER_NUMBER_OF_PAGES,
Integer.class);
+        if (numberOfPages == null) {
+            numberOfPages = te.getProperties().getNumberOfPages();
+        }
+
         Twitter twitter = te.getProperties().getTwitter();
         log.debug("Searching twitter with keywords: {}", keywords);
         QueryResult results = twitter.search(query);
         List<Status> list = results.getTweets();
 
+        for (int i = 1; i < numberOfPages; i++) {
+            if (results.hasNext() == false) {
+                break;
+            }
+            log.debug("Fetching page");
+            results = twitter.search(results.nextQuery());
+            list.addAll(results.getTweets());
+        }
+
         if (te.getProperties().isFilterOld()) {
             for (Status t : list) {
                 long newId = t.getId();

Modified: camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java
URL: http://svn.apache.org/viewvc/camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java?rev=1441326&r1=1441325&r2=1441326&view=diff
==============================================================================
--- camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java
(original)
+++ camel/trunk/components/camel-twitter/src/test/java/org/apache/camel/component/twitter/UriConfigurationTest.java
Fri Feb  1 07:42:13 2013
@@ -38,4 +38,14 @@ public class UriConfigurationTest extend
         Assert.assertTrue(!twitterEndpoint.getProperties().getAccessToken().isEmpty());
         Assert.assertTrue(!twitterEndpoint.getProperties().getAccessTokenSecret().isEmpty());
     }
+    
+    @Test
+    public void testPageSetting() throws Exception {
+        Endpoint endpoint = context.getEndpoint("twitter:todo/page?count=50&numberOfPages=2");
+        assertTrue("Endpoint not a TwitterEndpoint: " + endpoint, endpoint instanceof TwitterEndpoint);
+        TwitterEndpoint twitterEndpoint = (TwitterEndpoint) endpoint;
+
+        Assert.assertEquals(new Integer(50), twitterEndpoint.getProperties().getCount());
+        Assert.assertEquals(new Integer(2), twitterEndpoint.getProperties().getNumberOfPages());
+    }
 }



Mime
View raw message