streams-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mfranklin <...@git.apache.org>
Subject [GitHub] incubator-streams pull request: Streams 144
Date Wed, 20 Aug 2014 17:29:30 GMT
Github user mfranklin commented on a diff in the pull request:

    https://github.com/apache/incubator-streams/pull/69#discussion_r16490770
  
    --- Diff: streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/InstagramDataCollector.java
---
    @@ -0,0 +1,143 @@
    +/*
    +Licensed to the Apache Software Foundation (ASF) under one
    +or more contributor license agreements. See the NOTICE file
    +distributed with this work for additional information
    +regarding copyright ownership. The ASF licenses this file
    +to you under the Apache License, Version 2.0 (the
    +"License"); you may not use this file except in compliance *
    +http://www.apache.org/licenses/LICENSE-2.0 *
    +Unless required by applicable law or agreed to in writing,
    +software distributed under the License is distributed on an
    +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    +KIND, either express or implied. See the License for the
    +specific language governing permissions and limitations
    +under the License. */
    +package org.apache.streams.instagram.provider;
    +
    +import com.google.common.annotations.VisibleForTesting;
    +import org.apache.streams.core.StreamsDatum;
    +import org.apache.streams.instagram.InstagramConfiguration;
    +import org.apache.streams.instagram.User;
    +import org.apache.streams.util.ComponentUtils;
    +import org.apache.streams.util.api.requests.backoff.BackOffStrategy;
    +import org.apache.streams.util.api.requests.backoff.impl.ExponentialBackOffStrategy;
    +import org.apache.streams.util.oauth.tokens.tokenmanager.SimpleTokenManager;
    +import org.apache.streams.util.oauth.tokens.tokenmanager.impl.BasicTokenManger;
    +import org.jinstagram.Instagram;
    +import org.jinstagram.entity.common.Pagination;
    +import org.jinstagram.entity.users.feed.MediaFeed;
    +import org.jinstagram.entity.users.feed.MediaFeedData;
    +import org.jinstagram.exceptions.InstagramBadRequestException;
    +import org.jinstagram.exceptions.InstagramRateLimitException;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +import java.util.Collection;
    +import java.util.Queue;
    +import java.util.concurrent.atomic.AtomicBoolean;
    +
    +/**
    + * Executes on all of the Instagram requests to collect the Instagram data.
    + * <p/>
    + * If errors/exceptions occur when trying to gather data for a particular user, that
user is skipped and the collector
    + * move on to the next user.  If a rate limit exception occurs it employs an exponential
back off strategy.
    + */
    +public abstract class InstagramDataCollector<T> implements Runnable {
    +
    +    private static final Logger LOGGER = LoggerFactory.getLogger(InstagramDataCollector.class);
    +
    +    protected Queue<StreamsDatum> dataQueue; //exposed for testing
    +    private InstagramConfiguration config;
    +    private AtomicBoolean isCompleted;
    +    private SimpleTokenManager<InstagramOauthToken> tokenManger;
    +    protected int consecutiveErrorCount;
    +    protected BackOffStrategy backOffStrategy;
    +    private Instagram instagram;
    +
    +
    +    public InstagramDataCollector(Queue<StreamsDatum> queue, InstagramConfiguration
config) {
    +        this.dataQueue = queue;
    +        this.config = config;
    +        this.isCompleted = new AtomicBoolean(false);
    +        this.tokenManger = new BasicTokenManger<InstagramOauthToken>();
    +        for (String tokens : this.config.getUsersInfo().getAuthorizedTokens()) {
    +            this.tokenManger.addTokenToPool(new InstagramOauthToken(tokens));
    +        }
    +        this.consecutiveErrorCount = 0;
    +        this.backOffStrategy = new ExponentialBackOffStrategy(2);
    +        this.instagram = new Instagram(this.config.getClientId());
    +    }
    +
    +
    +    /**
    +     * If there are authorized tokens available, it sets a new token for the client and
returns
    +     * the client.  If there are no available tokens, it simply returns the client that
was
    +     * initialized in the constructor with client id.
    +     * @return
    +     */
    +    protected Instagram getNextInstagramClient() {
    +        if(this.tokenManger.numAvailableTokens() > 0) {
    +            this.instagram.setAccessToken(this.tokenManger.getNextAvailableToken());
    +        }
    +        return this.instagram;
    +    }
    +
    +    /**
    +     * Return the number of available tokens for this data collector
    +     * @return numbeer of available tokens
    +     */
    +    protected int numAvailableTokens() {
    +        return this.tokenManger.numAvailableTokens();
    +    }
    +
    +    /**
    +     * Queues the Instagram data to be output by the provider.
    +     * @param userData data to queue
    +     * @param userId user id who the data came from
    +     */
    +    protected void queueData(Collection<T> userData, String userId) {
    +        if (userData == null) {
    +            LOGGER.warn("User id, {}, returned a NULL data from instagram.", userId);
    +        } else {
    +            for (T data : userData) {
    +                ComponentUtils.offerUntilSuccess(convertToStreamsDatum(data), this.dataQueue);
    +            }
    +        }
    +    }
    +
    +    /**
    +     * @return true when the collector has queued all of the available Instagram data
for the provided users.
    +     */
    +    public boolean isCompleted() {
    +        return this.isCompleted.get();
    +    }
    +
    +    @Override
    +    public void run() {
    +        for (User user : this.config.getUsersInfo().getUsers()) {
    +            try {
    +                collectInstagramDataForUser(user);
    +            } catch (Exception e) {
    +                LOGGER.error("Exception thrown while polling for user, {}, skipping user.",
user.getUserId());
    --- End diff --
    
    If you get something like an InterruptedException, this will just catch it and move on.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message