beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lc...@apache.org
Subject [1/2] incubator-beam git commit: Remove overrides of isStreaming() and getAppName() in SparkPipelineOptions
Date Tue, 26 Jul 2016 02:17:56 GMT
Repository: incubator-beam
Updated Branches:
  refs/heads/master 7809f6bd2 -> 267136fb6


Remove overrides of isStreaming() and getAppName() in SparkPipelineOptions


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f52a656b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f52a656b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f52a656b

Branch: refs/heads/master
Commit: f52a656babc5de1dd8f9dc77bffc28c1726407a2
Parents: 7809f6b
Author: Pei He <peihe@google.com>
Authored: Fri Jul 22 16:49:34 2016 -0700
Committer: Luke Cwik <lcwik@google.com>
Committed: Mon Jul 25 19:16:26 2016 -0700

----------------------------------------------------------------------
 .../org/apache/beam/runners/spark/SparkPipelineOptions.java | 9 ---------
 .../beam/runners/spark/SparkStreamingPipelineOptions.java   | 9 ---------
 .../spark/translation/streaming/FlattenStreamingTest.java   | 2 +-
 .../spark/translation/streaming/KafkaStreamingTest.java     | 2 +-
 .../translation/streaming/SimpleStreamingWordCountTest.java | 2 +-
 5 files changed, 3 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
index 091382e..4bb2a57 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
@@ -32,14 +32,5 @@ public interface SparkPipelineOptions extends PipelineOptions, StreamingOptions,
   @Description("The url of the spark master to connect to, (e.g. spark://host:port, local[4]).")
   @Default.String("local[1]")
   String getSparkMaster();
-
   void setSparkMaster(String master);
-
-  @Override
-  @Default.Boolean(false)
-  boolean isStreaming();
-
-  @Override
-  @Default.String("spark dataflow pipeline job")
-  String getAppName();
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
index c789447..5944acd 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
@@ -28,14 +28,5 @@ public interface SparkStreamingPipelineOptions extends SparkPipelineOptions
{
           + "execution is stopped")
   @Default.Long(-1)
   Long getTimeout();
-
   void setTimeout(Long batchInterval);
-
-  @Override
-  @Default.Boolean(true)
-  boolean isStreaming();
-
-  @Override
-  @Default.String("spark streaming dataflow pipeline job")
-  String getAppName();
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
index 160f21d..ed7e9b7 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
@@ -59,8 +59,8 @@ public class FlattenStreamingTest {
   public void testRun() throws Exception {
     SparkStreamingPipelineOptions options =
         PipelineOptionsFactory.as(SparkStreamingPipelineOptions.class);
-    options.setAppName(this.getClass().getSimpleName());
     options.setRunner(SparkRunner.class);
+    options.setStreaming(true);
     options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
     Pipeline p = Pipeline.create(options);
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
index 5578e35..904b448 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
@@ -91,8 +91,8 @@ public class KafkaStreamingTest {
     // test read from Kafka
     SparkStreamingPipelineOptions options =
         PipelineOptionsFactory.as(SparkStreamingPipelineOptions.class);
-    options.setAppName(this.getClass().getSimpleName());
     options.setRunner(SparkRunner.class);
+    options.setStreaming(true);
     options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
     Pipeline p = Pipeline.create(options);
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
index 75ab274..4fa03f6 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
@@ -56,8 +56,8 @@ public class SimpleStreamingWordCountTest implements Serializable {
   public void testRun() throws Exception {
     SparkStreamingPipelineOptions options =
         PipelineOptionsFactory.as(SparkStreamingPipelineOptions.class);
-    options.setAppName(this.getClass().getSimpleName());
     options.setRunner(SparkRunner.class);
+    options.setStreaming(true);
     options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
     Pipeline p = Pipeline.create(options);
 


Mime
View raw message