streams-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sblack...@apache.org
Subject [27/41] git commit: moved json cleaner into streams-processor-jackson better javadoc comment
Date Thu, 04 Sep 2014 17:12:15 GMT
moved json cleaner into streams-processor-jackson
better javadoc comment


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/0536c98a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/0536c98a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/0536c98a

Branch: refs/heads/pp
Commit: 0536c98a845def2e09c7fa14ca8c0c4df0666505
Parents: 614f403
Author: sblackmon <sblackmon@apache.org>
Authored: Thu Aug 21 16:49:35 2014 -0500
Committer: sblackmon <sblackmon@apache.org>
Committed: Thu Aug 21 16:49:35 2014 -0500

----------------------------------------------------------------------
 streams-contrib/pom.xml                         |  1 +
 .../streams-processor-jackson/pom.xml           | 86 ++++++++++++++++++++
 .../CleanAdditionalPropertiesProcessor.java     | 62 ++++++++++++++
 .../streams-provider-datasift/pom.xml           |  5 ++
 .../CleanAdditionalPropertiesProcessor.java     | 62 --------------
 .../DatasiftTypeConverterProcessor.java         |  1 +
 6 files changed, 155 insertions(+), 62 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/pom.xml b/streams-contrib/pom.xml
index 699274e..e290466 100644
--- a/streams-contrib/pom.xml
+++ b/streams-contrib/pom.xml
@@ -48,6 +48,7 @@
         <!--<module>streams-processor-lucene</module>-->
         <!--<module>streams-processor-tika</module>-->
         <module>streams-provider-instagram</module>
+        <module>streams-processor-jackson</module>
         <module>streams-processor-json</module>
         <module>streams-processor-urls</module>
         <module>streams-provider-datasift</module>

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-processor-jackson/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-jackson/pom.xml b/streams-contrib/streams-processor-jackson/pom.xml
new file mode 100644
index 0000000..8bb44fd
--- /dev/null
+++ b/streams-contrib/streams-processor-jackson/pom.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>streams-processor-jackson</artifactId>
+    <version>0.1-SNAPSHOT</version>
+
+    <parent>
+        <groupId>org.apache.streams</groupId>
+        <artifactId>streams-contrib</artifactId>
+        <version>0.1-SNAPSHOT</version>
+    </parent>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.streams</groupId>
+            <artifactId>streams-config</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.streams</groupId>
+            <artifactId>streams-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.jayway.jsonpath</groupId>
+            <artifactId>json-path</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.jayway.jsonpath</groupId>
+            <artifactId>json-path-assert</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <sourceDirectory>src/main/java</sourceDirectory>
+        <testSourceDirectory>src/test/java</testSourceDirectory>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+        </resources>
+        <testResources>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
+        </testResources>
+        <plugins>
+
+        </plugins>
+    </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
new file mode 100644
index 0000000..56b0c5c
--- /dev/null
+++ b/streams-contrib/streams-processor-jackson/src/main/java/org/apache/streams/jackson/CleanAdditionalPropertiesProcessor.java
@@ -0,0 +1,62 @@
+package org.apache.streams.jackson;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This processor walks an input objectnode and corrects any artifacts
+ * that may have occured from improper serialization of jsonschema2pojo beans.
+ *
+ * The logic is also available for inclusion in other module via static import.
+ */
+public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(CleanAdditionalPropertiesProcessor.class);
+
+    private ObjectMapper mapper;
+
+    @Override
+    public List<StreamsDatum> process(StreamsDatum datum) {
+        List<StreamsDatum> result = Lists.newLinkedList();
+        ObjectNode activity = this.mapper.convertValue(datum.getDocument(), ObjectNode.class);
+        cleanAdditionalProperties(activity);
+        datum.setDocument(activity);
+        result.add(datum);
+        return result;
+    }
+
+    @Override
+    public void prepare(Object o) {
+        this.mapper = StreamsJacksonMapper.getInstance();
+        this.mapper.registerModule(new JsonOrgModule());
+    }
+
+    @Override
+    public void cleanUp() {
+
+    }
+
+    public static void cleanAdditionalProperties(ObjectNode node) {
+        if( node.get("additionalProperties") != null ) {
+            ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
+            cleanAdditionalProperties(additionalProperties);
+            Iterator<Map.Entry<String, JsonNode>> jsonNodeIterator = additionalProperties.fields();
+            while( jsonNodeIterator.hasNext() ) {
+                Map.Entry<String, JsonNode> entry = jsonNodeIterator.next();
+                node.put(entry.getKey(), entry.getValue());
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml
index f2aab08..782db07 100644
--- a/streams-contrib/streams-provider-datasift/pom.xml
+++ b/streams-contrib/streams-provider-datasift/pom.xml
@@ -37,6 +37,11 @@
         </dependency>
         <dependency>
             <groupId>org.apache.streams</groupId>
+            <artifactId>streams-processor-jackson</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.streams</groupId>
             <artifactId>streams-provider-twitter</artifactId>
             <version>${project.version}</version>
             <optional>true</optional>

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
deleted file mode 100644
index 00f08e7..0000000
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/CleanAdditionalPropertiesProcessor.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.apache.streams.datasift.processor;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule;
-import com.google.common.collect.Lists;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProcessor;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * HACK PROCESSOR.  Changes need to be made in apache streams to fix this issue long term.
- */
-public class CleanAdditionalPropertiesProcessor implements StreamsProcessor {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(CleanAdditionalPropertiesProcessor.class);
-
-    private static final String ADDITIONAL_PROPERTIES = "additionalProperties";
-    private static final String EXTENSIONS = "extensions";
-
-    private ObjectMapper mapper;
-
-    @Override
-    public List<StreamsDatum> process(StreamsDatum datum) {
-        List<StreamsDatum> result = Lists.newLinkedList();
-        ObjectNode activity = this.mapper.convertValue(datum.getDocument(), ObjectNode.class);
-        cleanAdditionalProperties(activity);
-        datum.setDocument(activity);
-        result.add(datum);
-        return result;
-    }
-
-    @Override
-    public void prepare(Object o) {
-        this.mapper = StreamsJacksonMapper.getInstance();
-        this.mapper.registerModule(new JsonOrgModule());
-    }
-
-    @Override
-    public void cleanUp() {
-
-    }
-
-    public static void cleanAdditionalProperties(ObjectNode node) {
-        if( node.get("additionalProperties") != null ) {
-            ObjectNode additionalProperties = (ObjectNode) node.get("additionalProperties");
-            cleanAdditionalProperties(additionalProperties);
-            Iterator<Map.Entry<String, JsonNode>> jsonNodeIterator = additionalProperties.fields();
-            while( jsonNodeIterator.hasNext() ) {
-                Map.Entry<String, JsonNode> entry = jsonNodeIterator.next();
-                node.put(entry.getKey(), entry.getValue());
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0536c98a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
index e85e8b1..a00cf23 100644
--- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
+++ b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/processor/DatasiftTypeConverterProcessor.java
@@ -28,6 +28,7 @@ import org.apache.streams.datasift.provider.DatasiftConverter;
 import org.apache.streams.datasift.serializer.DatasiftActivitySerializer;
 import org.apache.streams.datasift.util.StreamsDatasiftMapper;
 import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.jackson.CleanAdditionalPropertiesProcessor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 


Mime
View raw message