Return-Path: X-Original-To: apmail-streams-commits-archive@minotaur.apache.org Delivered-To: apmail-streams-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2C88511BEA for ; Thu, 4 Sep 2014 17:12:27 +0000 (UTC) Received: (qmail 85300 invoked by uid 500); 4 Sep 2014 17:12:27 -0000 Delivered-To: apmail-streams-commits-archive@streams.apache.org Received: (qmail 85279 invoked by uid 500); 4 Sep 2014 17:12:27 -0000 Mailing-List: contact commits-help@streams.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@streams.incubator.apache.org Delivered-To: mailing list commits@streams.incubator.apache.org Received: (qmail 85241 invoked by uid 99); 4 Sep 2014 17:12:27 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Sep 2014 17:12:27 +0000 X-ASF-Spam-Status: No, hits=-2001.7 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Thu, 04 Sep 2014 17:11:52 +0000 Received: (qmail 83007 invoked by uid 99); 4 Sep 2014 17:11:50 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Sep 2014 17:11:50 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 060B0A082A8; Thu, 4 Sep 2014 17:11:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sblackmon@apache.org To: commits@streams.incubator.apache.org Date: Thu, 04 Sep 2014 17:11:58 -0000 Message-Id: <205275bc24c84b7b8e08e514fcf294f4@git.apache.org> In-Reply-To: <1f0cf972e6c64f87936d0d9bca2bc4c6@git.apache.org> References: <1f0cf972e6c64f87936d0d9bca2bc4c6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [10/41] git commit: easier to use existing jackson-annotated beans X-Virus-Checked: Checked by ClamAV on apache.org easier to use existing jackson-annotated beans Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e6380ceb Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e6380ceb Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e6380ceb Branch: refs/heads/pp Commit: e6380cebed7f4ce8d211cc7e2e73d0080041cc71 Parents: 6381dc9 Author: sblackmon Authored: Fri Aug 8 14:50:27 2014 -0500 Committer: sblackmon Committed: Tue Aug 12 18:04:23 2014 -0500 ---------------------------------------------------------------------- .../provider/DatasiftManagedSourceSetup.java | 69 ++++++++++++++ .../provider/DatasiftStreamConfigurator.java | 10 ++- .../provider/DatasiftStreamProvider.java | 2 +- .../com/datasift/DatasiftConfiguration.json | 94 +------------------- 4 files changed, 79 insertions(+), 96 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e6380ceb/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java new file mode 100644 index 0000000..6e648e7 --- /dev/null +++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftManagedSourceSetup.java @@ -0,0 +1,69 @@ +package org.apache.streams.datasift.provider; + +import com.datasift.client.DataSiftClient; +import com.datasift.client.FutureData; +import com.datasift.client.managedsource.ManagedSource; +import com.datasift.client.managedsource.ManagedSourceList; +import com.datasift.client.managedsource.sources.DataSource; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.streams.StreamsConfiguration; +import org.apache.streams.config.StreamsConfigurator; +import org.apache.streams.datasift.DatasiftConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Created by sblackmon on 8/8/14. + */ +public class DatasiftManagedSourceSetup implements Runnable { + + private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamProvider.class); + + private static DatasiftConfiguration config = DatasiftStreamConfigurator.detectConfiguration(StreamsConfigurator.config); + + DataSiftClient client; + Map currentManagedSourceMap = Maps.newHashMap(); + List updatedManagedSourceList; + + public static void main(String[] args) { + DatasiftManagedSourceSetup job = new DatasiftManagedSourceSetup(); + (new Thread(job)).start(); + } + + @Override + public void run() { + + setup(); + + current(); + + updatedManagedSourceList = config.getManagedSources(); + + for( ManagedSource source : updatedManagedSourceList ) { + ManagedSource current = currentManagedSourceMap.get( source.getId() ); + ManagedSource updated = client.managedSource().update(current.getName(), (DataSource) source, current).sync(); + } + + } + + public void setup() { + client = DatasiftStreamProvider.getNewClient(config.getUserName(), config.getApiKey()); + } + + public void current() { + ManagedSourceList managedSources = client.managedSource().get().sync(); + Iterator managedSourceIterator = managedSources.iterator(); + while( managedSourceIterator.hasNext() ) { + ManagedSource source = managedSourceIterator.next(); + currentManagedSourceMap.put(source.getId(), source); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e6380ceb/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java index 7025d39..5726b4f 100644 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java +++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamConfigurator.java @@ -34,8 +34,14 @@ public class DatasiftStreamConfigurator { public static DatasiftConfiguration detectConfiguration(Config datasift) { - DatasiftConfiguration datasiftConfiguration; - datasiftConfiguration = StreamsJacksonMapper.getInstance().convertValue(datasift.root().render(ConfigRenderOptions.concise()), DatasiftConfiguration.class); + DatasiftConfiguration datasiftConfiguration = null; + + try { + datasiftConfiguration = mapper.readValue(datasift.root().render(ConfigRenderOptions.concise()), DatasiftConfiguration.class); + } catch (Exception e) { + e.printStackTrace(); + LOGGER.warn("Could not parse datasiftConfiguration"); + } return datasiftConfiguration; } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e6380ceb/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java index 8ed1443..c3f5ea3 100644 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java +++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java @@ -128,7 +128,7 @@ public class DatasiftStreamProvider implements StreamsProvider { * @param apiKey * @return */ - protected DataSiftClient getNewClient(String userName, String apiKey) { + protected static DataSiftClient getNewClient(String userName, String apiKey) { return new DataSiftClient(new DataSiftConfig(userName, apiKey)); } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e6380ceb/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json index ae9daa6..e024f43 100644 --- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json +++ b/streams-contrib/streams-provider-datasift/src/main/jsonschema/com/datasift/DatasiftConfiguration.json @@ -31,100 +31,8 @@ "type": "array", "minItems": 0, "items": { - "javaType": "org.apache.streams.datasift.managed.ManagedSource", "type": "object", - "properties": { - "auth": { - "type": "array", - "items": { - "javaType": "org.apache.streams.datasift.managed.ManagedSourceAuth", - "type": "object", - "properties": { - "identity_id": { - "type": "string" - }, - "parameters": { - "type": "object", - "properties": { - "value": { - "type": "string" - } - } - } - } - } - }, - "created_at": { - "type": "number" - }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parameters": { - "javaType": "org.apache.streams.datasift.managed.ManagedSourceParameters", - "type": "object", - "properties": { - "comments": { - "type": "boolean" - }, - "likes": { - "type": "boolean" - } - } - }, - "resources": { - "type": "array", - "items": [ - { - "javaType": "org.apache.streams.datasift.managed.ManagedSourceResource", - "type": "object", - "properties": { - "parameters": { - "type": "object", - "properties": { - "distance": { - "type": "string" - }, - "exact_match": { - "type": "string" - }, - "foursq": { - "type": "string" - }, - "lat": { - "type": "string" - }, - "lng": { - "type": "string" - }, - "type": { - "type": "string" - }, - "usermeta": { - "type": "string" - }, - "value": { - "type": "number" - } - } - }, - "resource_id": { - "type": "string" - } - } - } - ] - }, - "source_type": { - "type": "string" - }, - "status": { - "type": "string" - } - } + "javaType": "com.datasift.client.managedsource.ManagedSource" } } }