streams-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sblack...@apache.org
Subject incubator-streams-examples git commit: improve documentation
Date Fri, 30 Sep 2016 03:02:25 GMT
Repository: incubator-streams-examples
Updated Branches:
  refs/heads/flink 0112a8387 -> 58fefc07f


improve documentation


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/58fefc07
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/58fefc07
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/58fefc07

Branch: refs/heads/flink
Commit: 58fefc07fc09b45f0e4cfebc9b126cab1fa8a9a3
Parents: 0112a83
Author: Steve Blackmon @steveblackmon <sblackmon@apache.org>
Authored: Thu Sep 29 22:02:14 2016 -0500
Committer: Steve Blackmon @steveblackmon <sblackmon@apache.org>
Committed: Thu Sep 29 22:02:14 2016 -0500

----------------------------------------------------------------------
 .../resources/FlinkTwitterFollowingPipeline.dot | 37 ++++++++++++
 .../FlinkTwitterFollowingPipeline.dot.svg       | 63 ++++++++++++++++++++
 .../resources/FlinkTwitterPostsPipeline.dot     | 37 ++++++++++++
 .../resources/FlinkTwitterPostsPipeline.dot.svg | 63 ++++++++++++++++++++
 .../resources/FlinkTwitterSpritzerPipeline.dot  | 33 ++++++++++
 .../FlinkTwitterSpritzerPipeline.dot.svg        | 47 +++++++++++++++
 .../FlinkTwitterUserInformationPipeline.dot     | 37 ++++++++++++
 .../FlinkTwitterUserInformationPipeline.dot.svg | 63 ++++++++++++++++++++
 .../markdown/FlinkTwitterFollowingPipeline.md   |  4 +-
 .../site/markdown/FlinkTwitterPostsPipeline.md  |  2 +-
 .../markdown/FlinkTwitterSpritzerPipeline.md    | 41 +++++++++++++
 .../FlinkTwitterUserInformationPipeline.md      |  4 +-
 .../src/site/markdown/index.md                  | 16 ++++-
 .../test/FlinkTwitterFollowingPipelineIT.scala  |  4 +-
 14 files changed, 444 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot
b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot
new file mode 100644
index 0000000..ba5e60d
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ digraph g {
+
+  //source
+  source [label="source\nhdfs://${host}:${port}/${path}/${readerPath}",shape=tab];
+
+  //providers
+  TwitterFollowingProvider [label="TwitterFollowingProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java"];
+
+  //persisters
+  RollingFileSink [label="RollingFileSink",shape=ellipse];
+  
+   //data
+  destination [label="destination\nhdfs://${host}:${port}/${path}/${writerPath}",shape=tab];
+
+  //stream
+  TwitterFollowingProvider -> source [dir=back,style=dashed];
+  TwitterFollowingProvider -> RollingFileSink [label="String"];
+  RollingFileSink -> destination;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot.svg
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot.svg
b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot.svg
new file mode 100644
index 0000000..79bee38
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterFollowingPipeline.dot.svg
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.39.20160214.2135 (20160214.2135)
+ -->
+<!-- Title: g Pages: 1 -->
+<svg width="504pt" height="203pt"
+ viewBox="0.00 0.00 504.29 203.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 199)">
+<title>g</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-199 500.286,-199 500.286,4
-4,4"/>
+<!-- source -->
+<g id="node1" class="node">
+<title>source</title>
+<polygon fill="none" stroke="black" points="275.082,-109 11.9727,-109 11.9727,-113 -0.0273492,-113
-0.0273492,-73 275.082,-73 275.082,-109"/>
+<polyline fill="none" stroke="black" points="-0.0273492,-109 11.9727,-109 "/>
+<text text-anchor="middle" x="137.527" y="-93.8" font-family="Times,serif" font-size="14.00">source</text>
+<text text-anchor="middle" x="137.527" y="-79.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${readerPath}</text>
+</g>
+<!-- TwitterFollowingProvider -->
+<g id="node2" class="node">
+<title>TwitterFollowingProvider</title>
+<g id="a_node2"><a xlink:href="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java"
xlink:title="TwitterFollowingProvider">
+<ellipse fill="none" stroke="black" cx="200.527" cy="-177" rx="102.174" ry="18"/>
+<text text-anchor="middle" x="200.527" y="-172.8" font-family="Times,serif" font-size="14.00">TwitterFollowingProvider</text>
+</a>
+</g>
+</g>
+<!-- TwitterFollowingProvider&#45;&gt;source -->
+<g id="edge1" class="edge">
+<title>TwitterFollowingProvider&#45;&gt;source</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M181.043,-150.402C171.25,-137.034
159.635,-121.178 150.869,-109.212"/>
+<polygon fill="black" stroke="black" points="178.434,-152.763 187.167,-158.762 184.081,-148.626
178.434,-152.763"/>
+</g>
+<!-- RollingFileSink -->
+<g id="node3" class="node">
+<title>RollingFileSink</title>
+<ellipse fill="none" stroke="black" cx="359.527" cy="-91" rx="66.7358" ry="18"/>
+<text text-anchor="middle" x="359.527" y="-86.8" font-family="Times,serif" font-size="14.00">RollingFileSink</text>
+</g>
+<!-- TwitterFollowingProvider&#45;&gt;RollingFileSink -->
+<g id="edge2" class="edge">
+<title>TwitterFollowingProvider&#45;&gt;RollingFileSink</title>
+<path fill="none" stroke="black" d="M232.321,-159.803C257.806,-146.019 293.67,-126.621
320.824,-111.934"/>
+<polygon fill="black" stroke="black" points="322.54,-114.985 329.671,-107.149 319.21,-108.828
322.54,-114.985"/>
+<text text-anchor="middle" x="306.641" y="-129.8" font-family="Times,serif" font-size="14.00">String</text>
+</g>
+<!-- destination -->
+<g id="node4" class="node">
+<title>destination</title>
+<polygon fill="none" stroke="black" points="496.044,-36 235.01,-36 235.01,-40 223.01,-40
223.01,-0 496.044,-0 496.044,-36"/>
+<polyline fill="none" stroke="black" points="223.01,-36 235.01,-36 "/>
+<text text-anchor="middle" x="359.527" y="-20.8" font-family="Times,serif" font-size="14.00">destination</text>
+<text text-anchor="middle" x="359.527" y="-6.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${writerPath}</text>
+</g>
+<!-- RollingFileSink&#45;&gt;destination -->
+<g id="edge3" class="edge">
+<title>RollingFileSink&#45;&gt;destination</title>
+<path fill="none" stroke="black" d="M359.527,-72.9551C359.527,-64.8828 359.527,-55.1764
359.527,-46.1817"/>
+<polygon fill="black" stroke="black" points="363.027,-46.0903 359.527,-36.0904 356.027,-46.0904
363.027,-46.0903"/>
+</g>
+</g>
+</svg>

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot
b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot
new file mode 100644
index 0000000..1092ff4
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ digraph g {
+
+  //source
+  source [label="source\nhdfs://${host}:${port}/${path}/${readerPath}",shape=tab];
+
+  //providers
+  TwitterTimelineProvider [label="TwitterTimelineProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java"];
+
+  //persisters
+  RollingFileSink [label="RollingFileSink",shape=ellipse];
+  
+   //data
+  destination [label="destination\nhdfs://${host}:${port}/${path}/${writerPath}",shape=tab];
+
+  //stream
+  TwitterTimelineProvider -> source [dir=back,style=dashed];
+  TwitterTimelineProvider -> RollingFileSink [label="String"];
+  RollingFileSink -> destination;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot.svg
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot.svg
b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot.svg
new file mode 100644
index 0000000..5698c45
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterPostsPipeline.dot.svg
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.39.20160214.2135 (20160214.2135)
+ -->
+<!-- Title: g Pages: 1 -->
+<svg width="504pt" height="203pt"
+ viewBox="0.00 0.00 504.29 203.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 199)">
+<title>g</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-199 500.286,-199 500.286,4
-4,4"/>
+<!-- source -->
+<g id="node1" class="node">
+<title>source</title>
+<polygon fill="none" stroke="black" points="275.082,-109 11.9727,-109 11.9727,-113 -0.0273492,-113
-0.0273492,-73 275.082,-73 275.082,-109"/>
+<polyline fill="none" stroke="black" points="-0.0273492,-109 11.9727,-109 "/>
+<text text-anchor="middle" x="137.527" y="-93.8" font-family="Times,serif" font-size="14.00">source</text>
+<text text-anchor="middle" x="137.527" y="-79.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${readerPath}</text>
+</g>
+<!-- TwitterTimelineProvider -->
+<g id="node2" class="node">
+<title>TwitterTimelineProvider</title>
+<g id="a_node2"><a xlink:href="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java"
xlink:title="TwitterTimelineProvider">
+<ellipse fill="none" stroke="black" cx="200.527" cy="-177" rx="97.6831" ry="18"/>
+<text text-anchor="middle" x="200.527" y="-172.8" font-family="Times,serif" font-size="14.00">TwitterTimelineProvider</text>
+</a>
+</g>
+</g>
+<!-- TwitterTimelineProvider&#45;&gt;source -->
+<g id="edge1" class="edge">
+<title>TwitterTimelineProvider&#45;&gt;source</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M181.043,-150.402C171.25,-137.034
159.635,-121.178 150.869,-109.212"/>
+<polygon fill="black" stroke="black" points="178.434,-152.763 187.167,-158.762 184.081,-148.626
178.434,-152.763"/>
+</g>
+<!-- RollingFileSink -->
+<g id="node3" class="node">
+<title>RollingFileSink</title>
+<ellipse fill="none" stroke="black" cx="359.527" cy="-91" rx="66.7358" ry="18"/>
+<text text-anchor="middle" x="359.527" y="-86.8" font-family="Times,serif" font-size="14.00">RollingFileSink</text>
+</g>
+<!-- TwitterTimelineProvider&#45;&gt;RollingFileSink -->
+<g id="edge2" class="edge">
+<title>TwitterTimelineProvider&#45;&gt;RollingFileSink</title>
+<path fill="none" stroke="black" d="M232.321,-159.803C257.806,-146.019 293.67,-126.621
320.824,-111.934"/>
+<polygon fill="black" stroke="black" points="322.54,-114.985 329.671,-107.149 319.21,-108.828
322.54,-114.985"/>
+<text text-anchor="middle" x="306.641" y="-129.8" font-family="Times,serif" font-size="14.00">String</text>
+</g>
+<!-- destination -->
+<g id="node4" class="node">
+<title>destination</title>
+<polygon fill="none" stroke="black" points="496.044,-36 235.01,-36 235.01,-40 223.01,-40
223.01,-0 496.044,-0 496.044,-36"/>
+<polyline fill="none" stroke="black" points="223.01,-36 235.01,-36 "/>
+<text text-anchor="middle" x="359.527" y="-20.8" font-family="Times,serif" font-size="14.00">destination</text>
+<text text-anchor="middle" x="359.527" y="-6.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${writerPath}</text>
+</g>
+<!-- RollingFileSink&#45;&gt;destination -->
+<g id="edge3" class="edge">
+<title>RollingFileSink&#45;&gt;destination</title>
+<path fill="none" stroke="black" d="M359.527,-72.9551C359.527,-64.8828 359.527,-55.1764
359.527,-46.1817"/>
+<polygon fill="black" stroke="black" points="363.027,-46.0903 359.527,-36.0904 356.027,-46.0904
363.027,-46.0903"/>
+</g>
+</g>
+</svg>

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot
b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot
new file mode 100644
index 0000000..5a57595
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ digraph g {
+
+  //providers
+  TwitterStreamProvider [label="TwitterStreamProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java"];
+
+  //persisters
+  RollingFileSink [label="RollingFileSink",shape=ellipse];
+  
+   //data
+  destination [label="hdfs://${host}:${port}/${path}/${writerPath}",shape=box];
+
+  //stream
+  TwitterStreamProvider -> RollingFileSink [label="String"];
+  RollingFileSink -> destination;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot.svg
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot.svg
b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot.svg
new file mode 100644
index 0000000..960a11f
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterSpritzerPipeline.dot.svg
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.39.20160214.2135 (20160214.2135)
+ -->
+<!-- Title: g Pages: 1 -->
+<svg width="282pt" height="203pt"
+ viewBox="0.00 0.00 281.52 203.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 199)">
+<title>g</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-199 277.517,-199 277.517,4
-4,4"/>
+<!-- TwitterStreamProvider -->
+<g id="node1" class="node">
+<title>TwitterStreamProvider</title>
+<g id="a_node1"><a xlink:href="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java"
xlink:title="TwitterStreamProvider">
+<ellipse fill="none" stroke="black" cx="136.758" cy="-177" rx="91.0473" ry="18"/>
+<text text-anchor="middle" x="136.758" y="-172.8" font-family="Times,serif" font-size="14.00">TwitterStreamProvider</text>
+</a>
+</g>
+</g>
+<!-- RollingFileSink -->
+<g id="node2" class="node">
+<title>RollingFileSink</title>
+<ellipse fill="none" stroke="black" cx="136.758" cy="-91" rx="66.7358" ry="18"/>
+<text text-anchor="middle" x="136.758" y="-86.8" font-family="Times,serif" font-size="14.00">RollingFileSink</text>
+</g>
+<!-- TwitterStreamProvider&#45;&gt;RollingFileSink -->
+<g id="edge1" class="edge">
+<title>TwitterStreamProvider&#45;&gt;RollingFileSink</title>
+<path fill="none" stroke="black" d="M136.758,-158.762C136.758,-147.36 136.758,-132.434
136.758,-119.494"/>
+<polygon fill="black" stroke="black" points="140.258,-119.212 136.758,-109.212 133.258,-119.212
140.258,-119.212"/>
+<text text-anchor="middle" x="153.872" y="-129.8" font-family="Times,serif" font-size="14.00">String</text>
+</g>
+<!-- destination -->
+<g id="node3" class="node">
+<title>destination</title>
+<polygon fill="none" stroke="black" points="273.275,-36 0.241273,-36 0.241273,-0 273.275,-0
273.275,-36"/>
+<text text-anchor="middle" x="136.758" y="-13.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${writerPath}</text>
+</g>
+<!-- RollingFileSink&#45;&gt;destination -->
+<g id="edge2" class="edge">
+<title>RollingFileSink&#45;&gt;destination</title>
+<path fill="none" stroke="black" d="M136.758,-72.9551C136.758,-64.8828 136.758,-55.1764
136.758,-46.1817"/>
+<polygon fill="black" stroke="black" points="140.258,-46.0903 136.758,-36.0904 133.258,-46.0904
140.258,-46.0903"/>
+</g>
+</g>
+</svg>

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot
b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot
new file mode 100644
index 0000000..4a37234
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ digraph g {
+
+  //source
+  source [label="source\nhdfs://${host}:${port}/${path}/${readerPath}",shape=tab];
+
+  //providers
+  TwitterUserInformationProvider [label="TwitterUserInformationProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProvider.java"];
+
+  //persisters
+  RollingFileSink [label="RollingFileSink",shape=ellipse];
+  
+   //data
+  destination [label="destination\nhdfs://${host}:${port}/${path}/${writerPath}",shape=tab];
+
+  //stream
+  TwitterUserInformationProvider -> source [dir=back,style=dashed];
+  TwitterUserInformationProvider -> RollingFileSink [label="String"];
+  RollingFileSink -> destination;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot.svg
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot.svg
b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot.svg
new file mode 100644
index 0000000..9dadc63
--- /dev/null
+++ b/flink/flink-twitter-collection/src/main/resources/FlinkTwitterUserInformationPipeline.dot.svg
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.39.20160214.2135 (20160214.2135)
+ -->
+<!-- Title: g Pages: 1 -->
+<svg width="504pt" height="203pt"
+ viewBox="0.00 0.00 504.29 203.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 199)">
+<title>g</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-199 500.286,-199 500.286,4
-4,4"/>
+<!-- source -->
+<g id="node1" class="node">
+<title>source</title>
+<polygon fill="none" stroke="black" points="275.082,-109 11.9727,-109 11.9727,-113 -0.0273492,-113
-0.0273492,-73 275.082,-73 275.082,-109"/>
+<polyline fill="none" stroke="black" points="-0.0273492,-109 11.9727,-109 "/>
+<text text-anchor="middle" x="137.527" y="-93.8" font-family="Times,serif" font-size="14.00">source</text>
+<text text-anchor="middle" x="137.527" y="-79.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${readerPath}</text>
+</g>
+<!-- TwitterUserInformationProvider -->
+<g id="node2" class="node">
+<title>TwitterUserInformationProvider</title>
+<g id="a_node2"><a xlink:href="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterUserInformationProvider.java"
xlink:title="TwitterUserInformationProvider">
+<ellipse fill="none" stroke="black" cx="200.527" cy="-177" rx="124.824" ry="18"/>
+<text text-anchor="middle" x="200.527" y="-172.8" font-family="Times,serif" font-size="14.00">TwitterUserInformationProvider</text>
+</a>
+</g>
+</g>
+<!-- TwitterUserInformationProvider&#45;&gt;source -->
+<g id="edge1" class="edge">
+<title>TwitterUserInformationProvider&#45;&gt;source</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M181.043,-150.402C171.25,-137.034
159.635,-121.178 150.869,-109.212"/>
+<polygon fill="black" stroke="black" points="178.434,-152.763 187.167,-158.762 184.081,-148.626
178.434,-152.763"/>
+</g>
+<!-- RollingFileSink -->
+<g id="node3" class="node">
+<title>RollingFileSink</title>
+<ellipse fill="none" stroke="black" cx="359.527" cy="-91" rx="66.7358" ry="18"/>
+<text text-anchor="middle" x="359.527" y="-86.8" font-family="Times,serif" font-size="14.00">RollingFileSink</text>
+</g>
+<!-- TwitterUserInformationProvider&#45;&gt;RollingFileSink -->
+<g id="edge2" class="edge">
+<title>TwitterUserInformationProvider&#45;&gt;RollingFileSink</title>
+<path fill="none" stroke="black" d="M232.703,-159.597C258.14,-145.839 293.738,-126.584
320.747,-111.975"/>
+<polygon fill="black" stroke="black" points="322.42,-115.05 329.551,-107.214 319.09,-108.893
322.42,-115.05"/>
+<text text-anchor="middle" x="306.641" y="-129.8" font-family="Times,serif" font-size="14.00">String</text>
+</g>
+<!-- destination -->
+<g id="node4" class="node">
+<title>destination</title>
+<polygon fill="none" stroke="black" points="496.044,-36 235.01,-36 235.01,-40 223.01,-40
223.01,-0 496.044,-0 496.044,-36"/>
+<polyline fill="none" stroke="black" points="223.01,-36 235.01,-36 "/>
+<text text-anchor="middle" x="359.527" y="-20.8" font-family="Times,serif" font-size="14.00">destination</text>
+<text text-anchor="middle" x="359.527" y="-6.8" font-family="Times,serif" font-size="14.00">hdfs://${host}:${port}/${path}/${writerPath}</text>
+</g>
+<!-- RollingFileSink&#45;&gt;destination -->
+<g id="edge3" class="edge">
+<title>RollingFileSink&#45;&gt;destination</title>
+<path fill="none" stroke="black" d="M359.527,-72.9551C359.527,-64.8828 359.527,-55.1764
359.527,-46.1817"/>
+<polygon fill="black" stroke="black" points="363.027,-46.0903 359.527,-36.0904 356.027,-46.0904
363.027,-46.0903"/>
+</g>
+</g>
+</svg>

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
index 22f30f5..3ad23d3 100644
--- a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
+++ b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterFollowingPipeline.md
@@ -19,7 +19,9 @@ Diagram:
 Example Configuration:
 ----------------------
 
-[FlinkTwitterFollowingPipeline.json](FlinkTwitterFollowingPipeline.json "FlinkTwitterFollowingPipeline.json"
)
+[FlinkTwitterFollowingPipelineFollowersIT.conf](FlinkTwitterFollowingPipelineFollowersIT.conf
"FlinkTwitterFollowingPipelineFollowersIT.conf" )
+
+[FlinkTwitterFollowingPipelineFriendsIT.conf](FlinkTwitterFollowingPipelineFriendsIT.conf
"FlinkTwitterFollowingPipelineFriendsIT.conf" )
 
 Run (Local):
 ------------

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
index 5f77994..fe6b544 100644
--- a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
+++ b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterPostsPipeline.md
@@ -19,7 +19,7 @@ Diagram:
 Example Configuration:
 ----------------------
 
-[FlinkTwitterPostsPipeline.json](FlinkTwitterPostsPipeline.json "FlinkTwitterPostsPipeline.json"
)
+[FlinkTwitterPostsPipelineIT.conf](FlinkTwitterPostsPipelineIT.conf "FlinkTwitterPostsPipelineIT.conf"
)
 
 Run (Local):
 ------------

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterSpritzerPipeline.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterSpritzerPipeline.md
b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterSpritzerPipeline.md
new file mode 100644
index 0000000..259fe7f
--- /dev/null
+++ b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterSpritzerPipeline.md
@@ -0,0 +1,41 @@
+FlinkTwitterSpritzerPipeline
+============================
+
+Description:
+-----------------
+
+Collects twitter posts in real-time from the sample endpoint with flink.
+
+Specification:
+-----------------
+
+[FlinkTwitterSpritzerPipeline.dot](FlinkTwitterSpritzerPipeline.dot "FlinkTwitterSpritzerPipeline.dot"
)
+
+Diagram:
+-----------------
+
+![FlinkTwitterSpritzerPipeline.dot.svg](./FlinkTwitterSpritzerPipeline.dot.svg)
+
+Example Configuration:
+----------------------
+
+[FlinkTwitterSpritzerPipelineIT.conf](FlinkTwitterSpritzerPipelineIT.conf "FlinkTwitterSpritzerPipelineIT.conf"
)
+
+Run (Local):
+------------
+
+    java -cp dist/flink-twitter-collection-jar-with-dependencies.jar -Dconfig.file=file://<location_of_config_file>
org.apache.streams.examples.flink.twitter.collection.FlinkTwitterPostsPipeline
+
+Run (Flink):
+------------
+
+    flink-run.sh dist/flink-twitter-collection-jar-with-dependencies.jar org.apache.streams.examples.flink.twitter.collection.FlinkTwitterPostsPipeline
http://<location_of_config_file> 
+
+Run (YARN):
+-----------
+
+    flink-run.sh yarn dist/flink-twitter-collection-jar-with-dependencies.jar org.apache.streams.examples.flink.twitter.collection.FlinkTwitterPostsPipeline
http://<location_of_config_file> 
+
+[JavaDocs](apidocs/index.html "JavaDocs")
+
+###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
index 5e0d1fe..a465de9 100644
--- a/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
+++ b/flink/flink-twitter-collection/src/site/markdown/FlinkTwitterUserInformationPipeline.md
@@ -4,7 +4,7 @@ FlinkTwitterUserInformationPipeline
 Description:
 -----------------
 
-Collects twitter users with flink.
+Collects twitter user profiles with flink.
 
 Specification:
 -----------------
@@ -19,7 +19,7 @@ Diagram:
 Example Configuration:
 ----------------------
 
-[FlinkTwitterUserInformationPipeline.json](FlinkTwitterUserInformationPipeline.json "FlinkTwitterUserInformationPipeline.json"
)
+[FlinkTwitterUserInformationPipelineIT.conf](FlinkTwitterUserInformationPipelineIT.conf "FlinkTwitterUserInformationPipelineIT.conf"
)
 
 Run (Local):
 ------------

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/site/markdown/index.md b/flink/flink-twitter-collection/src/site/markdown/index.md
index 19e44cf..0f15603 100644
--- a/flink/flink-twitter-collection/src/site/markdown/index.md
+++ b/flink/flink-twitter-collection/src/site/markdown/index.md
@@ -22,10 +22,24 @@ Streams:
 
 <a href="FlinkTwitterFollowingPipeline.html" target="_self">FlinkTwitterFollowingPipeline</a>
 
+Test:
+-----
+
+Create a local file `application.conf` with valid twitter credentials
+
+    twitter {
+      oauth {
+        consumerKey = ""
+        consumerSecret = ""
+        accessToken = ""
+        accessTokenSecret = ""
+      }
+    }
+    
 Build:
 ---------
 
-    mvn clean install verify
+    mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf"
 
 [JavaDocs](apidocs/index.html "JavaDocs")
 

http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/58fefc07/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
----------------------------------------------------------------------
diff --git a/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
index b051e90..e6294f6 100644
--- a/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
+++ b/flink/flink-twitter-collection/src/test/scala/org/apache/streams/examples/flink/twitter/test/FlinkTwitterFollowingPipelineIT.scala
@@ -27,7 +27,7 @@ class FlinkTwitterFollowingPipelineIT extends FlatSpec {
 
   import FlinkTwitterFollowingPipeline._
 
-  @Test
+  @Test(enabled = false)
   def flinkTwitterFollowersPipelineFriendsIT = {
 
     val reference: Config = ConfigFactory.load()
@@ -55,7 +55,7 @@ class FlinkTwitterFollowingPipelineIT extends FlatSpec {
 
   }
 
-  @Test
+  @Test(enabled = false)
   def flinkTwitterFollowersPipelineFollowersIT = {
 
     val reference: Config = ConfigFactory.load()


Mime
View raw message