Return-Path: X-Original-To: apmail-streams-commits-archive@minotaur.apache.org Delivered-To: apmail-streams-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9D471CD63 for ; Sun, 16 Nov 2014 00:56:42 +0000 (UTC) Received: (qmail 97590 invoked by uid 500); 16 Nov 2014 00:56:42 -0000 Delivered-To: apmail-streams-commits-archive@streams.apache.org Received: (qmail 97569 invoked by uid 500); 16 Nov 2014 00:56:42 -0000 Mailing-List: contact commits-help@streams.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@streams.incubator.apache.org Delivered-To: mailing list commits@streams.incubator.apache.org Received: (qmail 97560 invoked by uid 99); 16 Nov 2014 00:56:42 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 16 Nov 2014 00:56:42 +0000 X-ASF-Spam-Status: No, hits=-2000.6 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Sun, 16 Nov 2014 00:56:40 +0000 Received: (qmail 96022 invoked by uid 99); 16 Nov 2014 00:56:20 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 16 Nov 2014 00:56:20 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 0D89082413E; Sun, 16 Nov 2014 00:56:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sblackmon@apache.org To: commits@streams.incubator.apache.org Date: Sun, 16 Nov 2014 00:56:19 -0000 Message-Id: <5824de44100b4d7797fee6a02f990bce@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/10] incubator-streams git commit: STREAMS-220 | Rewrote FacebookPostActivitySerializer to guarantee final Activity data shape X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-streams Updated Branches: refs/heads/STREAMS-68,218 8737bdadf -> ce2a794dc STREAMS-220 | Rewrote FacebookPostActivitySerializer to guarantee final Activity data shape Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/01efddab Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/01efddab Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/01efddab Branch: refs/heads/STREAMS-68,218 Commit: 01efddab57355035ea480046d10122fa22e60eff Parents: bfa9466 Author: Robert Douglas Authored: Wed Nov 12 12:29:55 2014 -0600 Committer: Robert Douglas Committed: Wed Nov 12 12:29:55 2014 -0600 ---------------------------------------------------------------------- .../api/FacebookPostActivitySerializer.java | 243 +------------------ .../serializer/FacebookActivityUtil.java | 147 ++++++++++- .../facebook/test/FacebookPostSerDeTest.java | 28 ++- 3 files changed, 163 insertions(+), 255 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/01efddab/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java index aff87eb..2bc0867 100644 --- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java +++ b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/api/FacebookPostActivitySerializer.java @@ -15,37 +15,22 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.streams.facebook.api; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.commons.lang.NotImplementedException; import org.apache.streams.data.ActivitySerializer; import org.apache.streams.exceptions.ActivitySerializerException; import org.apache.streams.facebook.Post; +import org.apache.streams.facebook.serializer.FacebookActivityUtil; import org.apache.streams.jackson.StreamsJacksonMapper; import org.apache.streams.pojo.json.*; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.ISODateTimeFormat; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import static org.apache.streams.data.util.ActivityUtil.*; - -/** - * Serializes activity posts - * sblackmon: This class needs a rewrite - */ public class FacebookPostActivitySerializer implements ActivitySerializer { public static final DateTimeFormatter FACEBOOK_FORMAT = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ssZ"); @@ -70,25 +55,10 @@ public class FacebookPostActivitySerializer implements ActivitySerializer links = Lists.newLinkedList(); - links.add(post.getLink()); - activity.setLinks(links); - ensureExtensions(activity).put("facebook", post); - if(post.getLikes() != null) { - Map likes = Maps.newHashMap(); - likes.put("count", post.getLikes().size()); - ensureExtensions(activity).put("likes", likes); - } + + FacebookActivityUtil.updateActivity(post, activity); + return activity; } @@ -96,209 +66,4 @@ public class FacebookPostActivitySerializer implements ActivitySerializer deserializeAll(List serializedList) { throw new NotImplementedException("Not currently supported by this deserializer"); } - - private void fixContentFromSummary(Activity activity) { - //we MUST have a content field set, so choose the best option - if(activity.getContent() == null) { - activity.setContent(activity.getAdditionalProperties().containsKey("summary") ? - (String) activity.getAdditionalProperties().get("summary") : - activity.getObject().getSummary()); - } - } - - private void fixObjectId(Activity activity) { - //An artifact of schema generation, the default value is {link} - if(activity.getObject().getId().equals("{link}")) { - activity.getObject().setId(null); - } - } - - private void setObjectType(String type, Activity activity) { - ActivityObject object = new ActivityObject(); - activity.setObject(object); - object.setObjectType(type); - } - - private void setProvider(Activity activity) { - Provider provider = new Provider(); - provider.setId("id:provider:"+PROVIDER_NAME); - provider.setDisplayName(PROVIDER_NAME); - activity.setProvider(provider); - } - - private String getObjectType(JsonNode node) { - Iterator> fields = node.fields(); - ensureMoreFields(fields); - Map.Entry field = fields.next(); - //ensureNoMoreFields(fields); - return node.asText(); - } - - private void parseObject(Activity activity, JsonNode jsonNode) throws ActivitySerializerException { - for(Iterator> fields = jsonNode.fields(); fields.hasNext();) { - Map.Entry field = fields.next(); - String key = field.getKey(); - JsonNode value = field.getValue(); - mapField(activity, key, value); - } - } - - private void mapField(Activity activity, String name, JsonNode value) throws ActivitySerializerException { - if("application".equals(name)) { - addGenerator(activity, value); - } else if ("caption".equals(name)) { - addSummary(activity, value); - } else if ("comments".equals(name)) { - addAttachments(activity, value); - } else if ("description".equals(name)) { - addObjectSummary(activity, value); - } else if ("from".equals(name)) { - addActor(activity, value); - } else if ("icon".equals(name)) { - addIcon(activity, value); - } else if ("id".equals(name)) { - addId(activity, value); - } else if ("is_hidden".equals(name)) { - addObjectHiddenExtension(activity, value); - } else if ("like_count".equals(name)) { - addLikeExtension(activity, value); - } else if ("link".equals(name)) { - addObjectLink(activity, value); - } else if ("message".equals(name)) { - activity.setContent(value.asText()); - } else if ("name".equals(name)) { - addObjectName(activity, value); - } else if ("object_id".equals(name)) { - addObjectId(activity, value); - } else if ("picture".equals(name)) { - addObjectImage(activity, value); - } else if ("place".equals(name)) { - addLocationExtension(activity, value); - } else if ("shares".equals(name)) { - addRebroadcastExtension(activity, value); - } else if ("source".equals(name)) { - addObjectLink(activity, value); - } else if ("story".equals(name)) { - addTitle(activity, value); - } - } - - private void addSummary(Activity activity, JsonNode value) { - activity.setAdditionalProperty("summary", value.asText()); - } - - private void addTitle(Activity activity, JsonNode value) { - activity.setTitle(value.asText()); - } - - private void addLikeExtension(Activity activity, JsonNode value) { - Map extensions = ensureExtensions(activity); - Map likes = Maps.newHashMap(); - likes.put("count", value.asLong()); - extensions.put(LIKES_EXTENSION, likes); - } - - private void addLocationExtension(Activity activity, JsonNode value) { - Map extensions = ensureExtensions(activity); - if(value.has("location")) { - Map location = new HashMap(); - JsonNode fbLocation = value.get("location"); - if(fbLocation.has("country")) { - location.put(LOCATION_EXTENSION_COUNTRY, fbLocation.get("country")); - } - if(fbLocation.has("latitude") && fbLocation.has("longitude")) { - location.put(LOCATION_EXTENSION_COORDINATES, String.format("%s,%s", fbLocation.get("longitude"), fbLocation.get("latitude"))); - } - extensions.put(LOCATION_EXTENSION, location); - } - } - - private void addObjectImage(Activity activity, JsonNode value) { - Image image = new Image(); - image.setUrl(value.asText()); - activity.getObject().setImage(image); - } - - private void addObjectId(Activity activity, JsonNode value) { - activity.getObject().setId(getObjectId("facebook", activity.getObject().getObjectType(), value.asText())); - } - - private void addObjectName(Activity activity, JsonNode value) { - activity.getObject().setDisplayName(value.asText()); - } - - private void addId(Activity activity, JsonNode value) { - activity.setId("id:"+PROVIDER_NAME+":"+value.asText()); - } - - private void addObjectLink(Activity activity, JsonNode value) { - activity.getObject().setUrl(value.asText()); - } - - private void addRebroadcastExtension(Activity activity, JsonNode value) { - Map extensions = ensureExtensions(activity); - if(value.has("count")) { - Map rebroadCast = Maps.newHashMap(); - rebroadCast.put("count", value.get("count").asLong()); - rebroadCast.put("perspectival", true); - extensions.put(REBROADCAST_EXTENSION, rebroadCast); - } - } - - private void addObjectHiddenExtension(Activity activity, JsonNode value) { - Map extensions = ensureExtensions(activity); - extensions.put("hidden", value.asBoolean()); - } - - private void addIcon(Activity activity, JsonNode value) { - Icon icon = new Icon(); - //Apparently the Icon didn't map from the schema very well - icon.setAdditionalProperty("url", value.asText()); - activity.setIcon(icon); - } - - private void addActor(Activity activity, JsonNode value) { - Actor actor = new Actor(); - if(value.has("name")) { - actor.setDisplayName(value.get("name").asText()); - } - if(value.has("id")) { - actor.setId("id:"+PROVIDER_NAME+":"+value.get("id").asText()); - } - activity.setActor(actor); - } - - private void addObjectSummary(Activity activity, JsonNode value) { - activity.getObject().setSummary(value.asText()); - } - - private void addGenerator(Activity activity, JsonNode value) { - Generator generator = new Generator(); - if(value.has("id")) { - generator.setId(getObjectId(PROVIDER_NAME, "generator", value.get("id").asText())); - } - if(value.has("name")) { - generator.setDisplayName(value.get("name").asText()); - } - if(value.has("namespace")) { - generator.setSummary(value.get("namespace").asText()); - } - activity.setGenerator(generator); - } - - private void addAttachments(Activity activity, JsonNode value) { - //No direct mapping at this time - } - - private static void ensureMoreFields(Iterator> fields) { - if(!fields.hasNext()) { - throw new IllegalStateException("Facebook activity must have one and only one root element"); - } - } - private static void ensureNoMoreFields(Iterator> fields) { - if(fields.hasNext()) { - throw new IllegalStateException("Facebook activity must have one and only one root element"); - } - } - } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/01efddab/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookActivityUtil.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookActivityUtil.java b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookActivityUtil.java index 06053b5..af650ba 100644 --- a/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookActivityUtil.java +++ b/streams-contrib/streams-provider-facebook/src/main/java/org/apache/streams/facebook/serializer/FacebookActivityUtil.java @@ -15,27 +15,29 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.streams.facebook.serializer; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.streams.exceptions.ActivitySerializerException; -import org.apache.streams.facebook.Cover; -import org.apache.streams.facebook.Location; -import org.apache.streams.facebook.Page; -import org.apache.streams.pojo.json.Activity; -import org.apache.streams.pojo.json.Actor; -import org.apache.streams.pojo.json.Image; -import org.apache.streams.pojo.json.Provider; +import org.apache.streams.facebook.*; +import org.apache.streams.facebook.Place; +import org.apache.streams.facebook.Post; +import org.apache.streams.pojo.json.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; +import java.util.List; import java.util.Map; -import static org.apache.streams.data.util.ActivityUtil.ensureExtensions; +import static org.apache.streams.data.util.ActivityUtil.*; public class FacebookActivityUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(FacebookActivityUtil.class); + /** * Updates the given Activity object with the values from the Page * @param page the object to use as the source @@ -49,6 +51,61 @@ public class FacebookActivityUtil { } /** + * Updates the given Activity object with the values from the Post + * @param post + * @param activity + * @throws ActivitySerializerException + */ + public static void updateActivity(Post post, Activity activity) throws ActivitySerializerException { + activity.setActor(buildActor(post)); + activity.setId(formatId(post.getId())); + activity.setProvider(getProvider()); + activity.setUpdated(post.getUpdatedTime()); + activity.setPublished(post.getCreatedTime()); + + if(post.getLink() != null && post.getLink().length() > 0) { + List links = Lists.newArrayList(); + links.add(post.getLink()); + activity.setLinks(links); + } + + activity.setContent(post.getMessage()); + + activity.setVerb("post"); + activity.setObject(buildObject(post)); + buildExtensions(activity, post); + } + + /** + * Builds out the {@link org.apache.streams.pojo.json.ActivityObject} from the given {@link org.apache.streams.pojo.json.Post} + * @param post + * @return {@link org.apache.streams.pojo.json.ActivityObject} + */ + public static ActivityObject buildObject(Post post) { + ActivityObject activityObject = new ActivityObject(); + + try { + activityObject.setContent(post.getMessage()); + activityObject.setPublished(post.getCreatedTime()); + activityObject.setUpdated(post.getUpdatedTime()); + activityObject.setDisplayName(post.getFrom().getName()); + activityObject.setId(formatId(post.getId())); + activityObject.setObjectType(post.getType()); + activityObject.setUrl(post.getLink()); + + if(activityObject.getObjectType().equals("photo")) { + Image image = new Image(); + image.setUrl(activityObject.getUrl()); + activityObject.setImage(image); + } + } catch (Exception e) { + LOGGER.error("Exception while trying to build Activity object for post: {}, exception: {}", post, e); + } + + return activityObject; + } + + /** * Gets the common facebook {@link org.apache.streams.pojo.json.Provider} object * @return a provider object representing Facebook */ @@ -95,6 +152,31 @@ public class FacebookActivityUtil { } /** + * Builds an {@link org.apache.streams.pojo.json.Actor} object from the {@link org.apache.streams.pojo.json.Post} + * @param post + * @return {@link org.apache.streams.pojo.json.Actor} + */ + public static Actor buildActor(Post post) { + Actor actor = new Actor(); + + try { + actor.setId(formatId( + Optional.fromNullable( + post.getFrom().getId()) + .or(Optional.of(post.getFrom().getId().toString())) + .orNull() + )); + + actor.setDisplayName(post.getFrom().getName()); + actor.setAdditionalProperty("handle", post.getFrom().getName()); + } catch (Exception e) { + LOGGER.error("Exception trying to build actor for Post: {}, {}", post, e); + } + + return actor; + } + + /** * Builds the actor extensions given the page object * @param actor * @param page @@ -115,6 +197,53 @@ public class FacebookActivityUtil { } /** + * Fills out the extensions attribute of the passed in {@link org.apache.streams.pojo.json.Activity} + * @param activity + * @param post + */ + public static void buildExtensions(Activity activity, Post post) { + Map extensions = ensureExtensions(activity); + + if(post.getLikes() != null) { + Map likes = Maps.newHashMap(); + org.apache.streams.facebook.Like like = post.getLikes().get(0); + + if(like.getAdditionalProperties().containsKey("data")) { + extensions.put("likes", likes); + } + } + + if(post.getShares() != null) { + Map shares = Maps.newHashMap(); + shares.put("count", ((Map)post.getShares()).get("count")); + extensions.put("rebroadcasts", shares); + } + + if(post.getTo() != null) { + To to = post.getTo(); + List data = to.getData(); + extensions.put("user_mentions", Lists.newArrayList()); + + for(Datum d : data) { + Map mention = Maps.newHashMap(); + + mention.put("id", d.getId()); + mention.put("displayName", d.getName()); + mention.put("handle", d.getName()); + + ((List>)extensions.get("user_mentions")).add(mention); + } + } + + if(post.getPlace() != null) { + Place place = post.getPlace(); + if(place.getAdditionalProperties().containsKey("location")) { + extensions.put(LOCATION_EXTENSION, place.getAdditionalProperties().get("location")); + } + } + } + + /** * Formats the ID to conform with the Apache Streams activity ID convention * @param idparts the parts of the ID to join * @return a valid Activity ID in format "id:facebook:part1:part2:...partN" http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/01efddab/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/FacebookPostSerDeTest.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/FacebookPostSerDeTest.java b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/FacebookPostSerDeTest.java index bbe2eeb..8a557d7 100644 --- a/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/FacebookPostSerDeTest.java +++ b/streams-contrib/streams-provider-facebook/src/test/java/org/apache/streams/facebook/test/FacebookPostSerDeTest.java @@ -24,7 +24,9 @@ import com.google.common.base.Joiner; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BoundedInputStream; import org.apache.streams.facebook.Post; +import org.apache.streams.facebook.serializer.FacebookActivityUtil; import org.apache.streams.jackson.StreamsJacksonMapper; +import org.apache.streams.pojo.json.Activity; import org.junit.Assert; import org.junit.Test; import org.junit.Ignore; @@ -33,13 +35,9 @@ import org.slf4j.LoggerFactory; import java.io.InputStream; -/** -* Created with IntelliJ IDEA. -* User: sblackmon -* Date: 8/20/13 -* Time: 5:57 PM -* To change this template use File | Settings | File Templates. -*/ +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + public class FacebookPostSerDeTest { private final static Logger LOGGER = LoggerFactory.getLogger(FacebookPostSerDeTest.class); @@ -73,6 +71,22 @@ public class FacebookPostSerDeTest { LOGGER.debug(mapper.writeValueAsString(serde)); + Activity activity = new Activity(); + FacebookActivityUtil.updateActivity(ser, activity); + + assertNotNull(activity); + assertNotNull(activity.getActor().getId()); + assertNotNull(activity.getActor().getDisplayName()); + assertNotNull(activity.getId()); + assert(activity.getVerb().equals("post")); + assertNotNull(activity.getObject()); + assertNotNull(activity.getUpdated()); + assertNotNull(activity.getPublished()); + assertEquals(activity.getProvider().getId(), "id:providers:facebook"); + assertEquals(activity.getProvider().getDisplayName(), "Facebook"); + assertEquals(activity.getLinks().size(), 1); + assertNotNull(activity.getAdditionalProperties().get("extensions")); + } catch( Exception e ) { System.out.println(e); e.printStackTrace();