streams-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sblack...@apache.org
Subject [1/2] incubator-streams git commit: resolves STREAMS-301 includes tests demonstrating concept of use
Date Tue, 31 Mar 2015 23:30:34 GMT
Repository: incubator-streams
Updated Branches:
  refs/heads/master 88bb08100 -> 96c140d1c


resolves STREAMS-301
includes tests demonstrating concept of use


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e41b7b1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e41b7b1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e41b7b1e

Branch: refs/heads/master
Commit: e41b7b1e58e130eaa535a68a64e7a06e1529425c
Parents: b99b546
Author: Steve Blackmon (@steveblackmon) <sblackmon@apache.org>
Authored: Sat Mar 28 13:24:04 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sblackmon@apache.org>
Committed: Sat Mar 28 13:24:04 2015 -0500

----------------------------------------------------------------------
 .../converter/HoconConverterProcessor.java      |  76 ++++++++++++
 .../streams/converter/HoconConverterUtil.java   | 101 ++++++++++++++++
 .../test/HoconConverterProcessorTest.java       | 117 +++++++++++++++++++
 .../converter/test/HoconConverterTest.java      |  94 +++++++++++++++
 .../src/test/resources/test1.conf               |   2 +
 .../src/test/resources/test2.conf               |   4 +
 .../src/test/resources/test3a.conf              |   4 +
 .../src/test/resources/test3b.conf              |   2 +
 8 files changed, 400 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
new file mode 100644
index 0000000..26b2efa
--- /dev/null
+++ b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterProcessor.java
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+package org.apache.streams.converter;
+
+import com.google.common.collect.Lists;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.core.util.DatumUtils;
+import org.apache.streams.pojo.json.Activity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * HoconConverterProcessor is a utility processor for converting any datum document
+ * with translation rules expressed as HOCON in the classpath or at a URL.
+ *
+ * To use this capability without a dedicated stream processor, just use HoconConverterUtil.
+ */
+public class HoconConverterProcessor implements StreamsProcessor {
+
+    private final static Logger LOGGER = LoggerFactory.getLogger(HoconConverterProcessor.class);
+
+    protected Class outClass;
+    protected String hocon;
+    protected String outPath;
+
+    public HoconConverterProcessor(Class outClass, String hocon, String outPath) {
+        this.outClass = outClass;
+        this.hocon = hocon;
+        this.outPath = outPath;
+    }
+
+    @Override
+    public List<StreamsDatum> process(StreamsDatum entry) {
+
+        List<StreamsDatum> result = Lists.newLinkedList();
+        Object document = entry.getDocument();
+
+        Object outDoc = HoconConverterUtil.convert(document, outClass, hocon, outPath);
+
+        StreamsDatum datum = DatumUtils.cloneDatum(entry);
+        datum.setDocument(outDoc);
+        result.add(datum);
+
+        return result;
+    }
+
+    @Override
+    public void prepare(Object configurationObject) {
+
+    }
+
+    @Override
+    public void cleanUp() {
+
+    }
+};

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
new file mode 100644
index 0000000..7a40606
--- /dev/null
+++ b/streams-components/streams-converters/src/main/java/org/apache/streams/converter/HoconConverterUtil.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.converter;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigRenderOptions;
+import com.typesafe.config.ConfigValue;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * HoconConverterUtil supports HoconConverterProcessor in converting types via application
+ * of hocon (https://github.com/typesafehub/config/blob/master/HOCON.md) scripts
+ */
+public class HoconConverterUtil {
+
+    private final static Logger LOGGER = LoggerFactory.getLogger(HoconConverterUtil.class);
+
+    private static ObjectMapper mapper = StreamsJacksonMapper.getInstance();
+
+    public static Object convert(Object object, Class outClass, String hoconResource) {
+        Config hocon = ConfigFactory.parseResources(hoconResource);
+        return convert(object, outClass, hocon);
+    }
+
+    public static Object convert(Object object, Class outClass, String hoconResource, String
outPath) {
+        Config hocon = ConfigFactory.parseResources(hoconResource);
+        return convert(object, outClass, hocon, outPath);
+    }
+
+    public static Object convert(Object object, Class outClass, Config hocon) {
+        return convert(object, outClass, hocon, null);
+    }
+
+    public static Object convert(Object object, Class outClass, Config hocon, String outPath)
{
+        String json = null;
+        Object outDoc = null;
+        if( object instanceof String ) {
+            json = (String) object;
+        } else {
+            try {
+                json = mapper.writeValueAsString(object);
+            } catch (JsonProcessingException e) {
+                LOGGER.warn("Failed to process input:", object);
+                return outDoc;
+            }
+        }
+
+        Config base = ConfigFactory.parseString(json);
+        Config converted = hocon.withFallback(base);
+
+        String outJson = null;
+        try {
+            if( outPath == null )
+                outJson = converted.resolve().root().render(ConfigRenderOptions.concise());
+            else {
+                Config resolved = converted.resolve();
+                ConfigObject outObject = resolved.withOnlyPath(outPath).root();
+                ConfigValue outValue = outObject.get(outPath);
+                outJson = outValue.render(ConfigRenderOptions.concise());
+            }
+        } catch (Exception e) {
+            LOGGER.warn("Failed to convert:", json);
+            LOGGER.warn(e.getMessage());
+        }
+        if( outClass == String.class )
+            return outJson;
+        else {
+            try {
+                outDoc = mapper.readValue( outJson, outClass );
+            } catch (IOException e) {
+                LOGGER.warn("Failed to convert:", object);
+            }
+        }
+        return outDoc;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
new file mode 100644
index 0000000..08b2332
--- /dev/null
+++ b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterProcessorTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.converter.test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
+import org.apache.streams.converter.HoconConverterProcessor;
+import org.apache.streams.converter.HoconConverterUtil;
+import org.apache.streams.converter.TypeConverterProcessor;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.data.util.ActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static junit.framework.Assert.*;
+
+/**
+ * Tests for
+ * @see {@link HoconConverterProcessor}
+ */
+public class HoconConverterProcessorTest {
+
+    /**
+     * Tests in-place simple conversion from String to String
+     */
+    @Test
+    public void testHoconConverter1() {
+
+        final String TEST_JSON_1 = "{\"race\":\"klingon\",\"gender\":\"male\"}";
+
+        StreamsProcessor processor = new HoconConverterProcessor(String.class, "test1.conf",
null);
+        processor.prepare(null);
+        StreamsDatum datum = new StreamsDatum(TEST_JSON_1, "1");
+        List<StreamsDatum> result = processor.process(datum);
+        assertNotNull(result);
+        assertEquals(1, result.size());
+        StreamsDatum resultDatum = result.get(0);
+        assertTrue(resultDatum.getDocument() instanceof String);
+        String result1 = (String) resultDatum.getDocument();
+        assertTrue(result1.contains("race"));
+        assertTrue(result1.contains("18"));
+        assertTrue(result1.contains("female"));
+    }
+
+    /**
+     * Tests derived object substitution conversion from String to ObjectNode
+     */
+    @Test
+    public void testHoconConverter2() {
+
+        final String TEST_ID_2 = "2";
+        final String TEST_JSON_2 = "{\"race\":\"klingon\",\"gender\":\"male\",\"age\":18}";
+
+        StreamsProcessor processor = new HoconConverterProcessor(ObjectNode.class, "test2.conf",
"demographics");
+        processor.prepare(null);
+        StreamsDatum datum = new StreamsDatum(TEST_JSON_2, TEST_ID_2);
+        List<StreamsDatum> result2 = processor.process(datum);
+        assertNotNull(result2);
+        assertEquals(1, result2.size());
+        StreamsDatum resultDatum = result2.get(0);
+        assertTrue(resultDatum.getDocument() instanceof ObjectNode);
+        assertTrue(resultDatum.getId().equals(TEST_ID_2));
+        ObjectNode resultDoc = (ObjectNode) resultDatum.getDocument();
+        assertNotNull(resultDoc);
+        assertTrue(resultDoc.get("race") != null);
+        assertTrue(resultDoc.get("age").asDouble() == 18);
+        assertTrue(resultDoc.get("gender").asText().equals("female"));
+    }
+
+    /**
+     * Tests derived object import conversion from String to Activity
+     */
+    @Test
+    public void testHoconConverter3() {
+
+        final String TEST_JSON_3 = "{\"id\":\"123\",\"text\":\"buncha stuff\",\"user\":{\"name\":\"guy\"}}";
+
+        StreamsProcessor processor = new HoconConverterProcessor(Activity.class, "test3a.conf",
"activity");
+        processor.prepare(null);
+        StreamsDatum datum = new StreamsDatum(TEST_JSON_3, "3");
+        List<StreamsDatum> result = processor.process(datum);
+        assertNotNull(result);
+        assertEquals(1, result.size());
+        StreamsDatum resultDatum = result.get(0);
+        assertTrue(resultDatum.getDocument() instanceof Activity);
+        Activity result3 = (Activity) resultDatum.getDocument();
+        assertNotNull(result3);
+        assertTrue(result3.getProvider() != null);
+        assertTrue(result3.getId().equals("id:123"));
+        assertTrue(result3.getContent().endsWith("stuff"));
+        assertTrue(result3.getActor().getDisplayName().equals("Jorge"));
+
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
new file mode 100644
index 0000000..8628e98
--- /dev/null
+++ b/streams-components/streams-converters/src/test/java/org/apache/streams/converter/test/HoconConverterTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.streams.converter.test;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
+import org.apache.streams.converter.HoconConverterUtil;
+import org.apache.streams.converter.TypeConverterProcessor;
+import org.apache.streams.core.StreamsDatum;
+import org.apache.streams.core.StreamsProcessor;
+import org.apache.streams.data.util.ActivityUtil;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.pojo.json.Activity;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static junit.framework.Assert.*;
+
+/**
+ * Tests for
+ * @see {@link HoconConverterUtil}
+ */
+public class HoconConverterTest {
+
+    /**
+     * Tests in-place simple conversion from String to String
+     */
+    @Test
+    public void testHoconConverter1() {
+
+        final String TEST_JSON_1 = "{\"race\":\"klingon\",\"gender\":\"male\"}";
+
+        String result1 = (String) HoconConverterUtil.convert(TEST_JSON_1, String.class, "test1.conf");
+
+        assertNotNull(result1);
+        assertTrue(result1.contains("race"));
+        assertTrue(result1.contains("18"));
+        assertTrue(result1.contains("female"));
+    }
+
+    /**
+     * Tests derived object substitution conversion from String to ObjectNode
+     */
+    @Test
+    public void testHoconConverter2() {
+
+        final String TEST_JSON_2 = "{\"race\":\"klingon\",\"gender\":\"male\",\"age\":18}";
+
+        ObjectNode result2 = (ObjectNode) HoconConverterUtil.convert(TEST_JSON_2, ObjectNode.class,
"test2.conf", "demographics");
+
+        assertNotNull(result2);
+        assertTrue(result2.get("race") != null);
+        assertTrue(result2.get("age").asDouble() == 18);
+        assertTrue(result2.get("gender").asText().equals("female"));
+    }
+
+    /**
+     * Tests derived object import conversion from String to Activity
+     */
+    @Test
+    public void testHoconConverter3() {
+
+        final String TEST_JSON_3 = "{\"id\":\"123\",\"text\":\"buncha stuff\",\"user\":{\"name\":\"guy\"}}";
+
+        Activity result3 = (Activity) HoconConverterUtil.convert(TEST_JSON_3, Activity.class,
"test3a.conf", "activity");
+
+        assertNotNull(result3);
+        assertTrue(result3.getProvider() != null);
+        assertTrue(result3.getId().equals("id:123"));
+        assertTrue(result3.getContent().endsWith("stuff"));
+        assertTrue(result3.getActor().getDisplayName().equals("Jorge"));
+
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/resources/test1.conf
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/resources/test1.conf b/streams-components/streams-converters/src/test/resources/test1.conf
new file mode 100644
index 0000000..ae8056c
--- /dev/null
+++ b/streams-components/streams-converters/src/test/resources/test1.conf
@@ -0,0 +1,2 @@
+age = 18
+gender = female
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/resources/test2.conf
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/resources/test2.conf b/streams-components/streams-converters/src/test/resources/test2.conf
new file mode 100644
index 0000000..b3e2cb4
--- /dev/null
+++ b/streams-components/streams-converters/src/test/resources/test2.conf
@@ -0,0 +1,4 @@
+demographics.age = 17
+demographics.age = ${?age}
+demographics.race = ${race}
+demographics.gender = fe${gender}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/resources/test3a.conf
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/resources/test3a.conf b/streams-components/streams-converters/src/test/resources/test3a.conf
new file mode 100644
index 0000000..4690e37
--- /dev/null
+++ b/streams-components/streams-converters/src/test/resources/test3a.conf
@@ -0,0 +1,4 @@
+activity.provider.id = test
+activity.id = "id:"${id}
+activity.content = ${text}
+activity.actor = {include "test3b.conf"}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/e41b7b1e/streams-components/streams-converters/src/test/resources/test3b.conf
----------------------------------------------------------------------
diff --git a/streams-components/streams-converters/src/test/resources/test3b.conf b/streams-components/streams-converters/src/test/resources/test3b.conf
new file mode 100644
index 0000000..b53d275
--- /dev/null
+++ b/streams-components/streams-converters/src/test/resources/test3b.conf
@@ -0,0 +1,2 @@
+id = "actorid"
+displayName = "Jorge"


Mime
View raw message