beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [13/39] incubator-beam git commit: Merge branch 'master' into apex-runner
Date Sat, 12 Nov 2016 02:28:31 GMT
Merge branch 'master' into apex-runner


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6a971d6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6a971d6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6a971d6b

Branch: refs/heads/master
Commit: 6a971d6b13a035f74ce1b2d2efc6d37f84a2373c
Parents: c9f1406 7de8d57
Author: Thomas Weise <thw@apache.org>
Authored: Tue Oct 25 09:07:50 2016 -0700
Committer: Thomas Weise <thw@apache.org>
Committed: Tue Oct 25 09:07:50 2016 -0700

----------------------------------------------------------------------
 .travis.yml                                     |  10 +-
 examples/java/pom.xml                           |   2 +-
 .../beam/examples/DebuggingWordCount.java       |   2 +-
 .../apache/beam/examples/WindowedWordCount.java |   2 +-
 .../org/apache/beam/examples/WordCount.java     |   4 +-
 .../common/ExampleBigQueryTableOptions.java     |   2 +-
 ...xamplePubsubTopicAndSubscriptionOptions.java |   2 +-
 .../common/ExamplePubsubTopicOptions.java       |   2 +-
 .../beam/examples/complete/AutoComplete.java    |   2 +-
 .../apache/beam/examples/complete/TfIdf.java    |   2 +-
 .../examples/complete/TopWikipediaSessions.java |   2 +-
 .../examples/cookbook/BigQueryTornadoes.java    |   2 +-
 .../cookbook/CombinePerKeyExamples.java         |   2 +-
 .../beam/examples/cookbook/DeDupExample.java    |   4 +-
 .../beam/examples/cookbook/FilterExamples.java  |   2 +-
 .../beam/examples/cookbook/JoinExamples.java    |   2 +-
 .../examples/cookbook/MaxPerKeyExamples.java    |   2 +-
 .../beam/examples/WindowedWordCountIT.java      |  11 +
 .../examples/cookbook/BigQueryTornadoesIT.java  |  14 +-
 examples/java8/pom.xml                          |   2 +-
 .../beam/examples/complete/game/GameStats.java  |   2 +-
 .../examples/complete/game/HourlyTeamScore.java |   2 +-
 .../examples/complete/game/LeaderBoard.java     |   2 +-
 .../beam/examples/complete/game/UserScore.java  |   2 +-
 examples/pom.xml                                |   2 +-
 pom.xml                                         |  17 +-
 runners/core-java/pom.xml                       |   2 +-
 .../beam/runners/core/AggregatorFactory.java    |  39 ++
 .../apache/beam/runners/core/DoFnRunner.java    |   6 +-
 .../apache/beam/runners/core/DoFnRunners.java   | 191 ++++--
 .../runners/core/ElementByteSizeObservable.java |   5 +-
 .../runners/core/GBKIntoKeyedWorkItems.java     |  17 +-
 .../core/GroupAlsoByWindowViaWindowSetDoFn.java |   5 +
 .../GroupAlsoByWindowsViaOutputBufferDoFn.java  |   5 +
 .../runners/core/ReduceFnContextFactory.java    |   2 +-
 .../beam/runners/core/ReduceFnRunner.java       |  39 +-
 .../beam/runners/core/SimpleDoFnRunner.java     | 337 ++++++----
 .../beam/runners/core/SimpleOldDoFnRunner.java  | 521 +++++++++++++++
 .../apache/beam/runners/core/TriggerRunner.java | 247 -------
 .../core/UnboundedReadFromBoundedSource.java    |   2 +-
 .../core/triggers/AfterAllStateMachine.java     |   5 +
 .../AfterDelayFromFirstElementStateMachine.java |  17 +-
 .../core/triggers/AfterEachStateMachine.java    |   5 +
 .../core/triggers/AfterFirstStateMachine.java   |   6 +
 .../core/triggers/AfterPaneStateMachine.java    |   7 +
 .../triggers/AfterWatermarkStateMachine.java    |   4 +-
 .../runners/core/triggers/FinishedTriggers.java |   8 +-
 .../core/triggers/TriggerStateMachine.java      |   4 +-
 .../triggers/TriggerStateMachineRunner.java     |   2 +-
 .../core/triggers/TriggerStateMachines.java     | 215 +++++++
 .../beam/runners/core/ReduceFnRunnerTest.java   | 281 ++++----
 .../beam/runners/core/ReduceFnTester.java       | 157 +++--
 .../beam/runners/core/SimpleDoFnRunnerTest.java |  88 ---
 .../runners/core/SimpleOldDoFnRunnerTest.java   |  88 +++
 .../beam/runners/core/SplittableParDoTest.java  |  14 +-
 .../core/triggers/TriggerStateMachinesTest.java | 199 ++++++
 runners/direct-java/pom.xml                     |  18 +-
 .../direct/BoundedReadEvaluatorFactory.java     |   3 +-
 .../beam/runners/direct/DirectMetrics.java      |   7 +
 .../beam/runners/direct/DirectRunner.java       |   2 +-
 .../runners/direct/DoFnLifecycleManager.java    |  42 +-
 .../runners/direct/ForwardingPTransform.java    |   2 +-
 .../beam/runners/direct/ParDoEvaluator.java     |   4 +-
 .../direct/ParDoMultiEvaluatorFactory.java      |   6 +-
 .../direct/ParDoSingleEvaluatorFactory.java     |   5 +-
 .../direct/TestStreamEvaluatorFactory.java      |   2 +-
 .../direct/UnboundedReadEvaluatorFactory.java   |  15 +-
 .../beam/runners/direct/WatermarkManager.java   |   4 +-
 .../beam/runners/direct/DirectRunnerTest.java   |  32 -
 ...leManagerRemovingTransformEvaluatorTest.java |  16 +-
 .../direct/DoFnLifecycleManagerTest.java        |  12 +-
 .../direct/DoFnLifecycleManagersTest.java       |  48 +-
 .../direct/ForwardingPTransformTest.java        |   7 +-
 .../direct/ParDoMultiEvaluatorFactoryTest.java  |  11 +
 .../direct/ParDoSingleEvaluatorFactoryTest.java |  11 +
 .../beam/runners/direct/SplittableDoFnTest.java |   8 +-
 .../UnboundedReadEvaluatorFactoryTest.java      |  52 ++
 .../runners/direct/WatermarkManagerTest.java    |   2 +-
 runners/flink/examples/pom.xml                  |   2 +-
 runners/flink/pom.xml                           |   2 +-
 runners/flink/runner/pom.xml                    |   2 +-
 .../flink/FlinkDetachedRunnerResult.java        |  76 +++
 .../apache/beam/runners/flink/FlinkRunner.java  |   9 +-
 .../beam/runners/flink/FlinkRunnerResult.java   |  11 +-
 .../beam/runners/flink/TestFlinkRunner.java     |   9 +-
 .../functions/FlinkProcessContext.java          |  10 +-
 runners/google-cloud-dataflow-java/pom.xml      |   4 +-
 .../dataflow/DataflowPipelineTranslator.java    | 133 ++--
 .../beam/runners/dataflow/DataflowRunner.java   |  20 +-
 .../DataflowUnboundedReadFromBoundedSource.java |   4 +-
 .../dataflow/internal/ReadTranslator.java       |   2 +-
 .../options/DataflowPipelineDebugOptions.java   |   4 +-
 .../options/DataflowPipelineOptions.java        |   2 +-
 .../DataflowPipelineWorkerPoolOptions.java      |   8 +-
 .../options/DataflowProfilingOptions.java       |   2 +-
 .../options/DataflowWorkerLoggingOptions.java   |   4 +-
 .../beam/runners/dataflow/util/DoFnInfo.java    |  30 +-
 .../runners/dataflow/util/MonitoringUtil.java   |   2 +-
 .../beam/runners/dataflow/util/Stager.java      |   2 +-
 .../DataflowPipelineTranslatorTest.java         |  38 +-
 .../runners/dataflow/DataflowRunnerTest.java    |   6 +-
 .../dataflow/util/MonitoringUtilTest.java       |   4 +-
 runners/pom.xml                                 |   3 +-
 runners/spark/pom.xml                           |  75 ++-
 .../runners/spark/SparkPipelineOptions.java     |   6 +-
 .../apache/beam/runners/spark/SparkRunner.java  |  48 +-
 .../beam/runners/spark/TestSparkRunner.java     |   4 +-
 .../beam/runners/spark/examples/WordCount.java  |   2 +-
 .../apache/beam/runners/spark/io/SourceRDD.java |   6 +-
 .../runners/spark/translation/DoFnFunction.java |  74 +--
 .../spark/translation/EvaluationContext.java    |   6 +-
 .../translation/GroupCombineFunctions.java      | 235 ++++---
 .../spark/translation/MultiDoFnFunction.java    |  85 ++-
 .../translation/SparkAbstractCombineFn.java     | 134 ++++
 .../spark/translation/SparkContextFactory.java  |   3 +-
 .../spark/translation/SparkGlobalCombineFn.java | 260 ++++++++
 .../spark/translation/SparkKeyedCombineFn.java  | 273 ++++++++
 .../spark/translation/SparkProcessContext.java  | 160 ++++-
 .../spark/translation/TransformTranslator.java  | 143 ++--
 .../spark/translation/TranslationUtils.java     |  28 +-
 .../streaming/StreamingEvaluationContext.java   |  33 +-
 .../streaming/StreamingTransformTranslator.java | 126 ++--
 .../runners/spark/util/BroadcastHelper.java     |  26 -
 .../spark/util/SparkSideInputReader.java        |  95 +++
 .../apache/beam/runners/spark/DeDupTest.java    |  58 --
 .../beam/runners/spark/EmptyInputTest.java      |  74 ---
 .../beam/runners/spark/SimpleWordCountTest.java | 107 ---
 .../apache/beam/runners/spark/TfIdfTest.java    | 259 --------
 .../spark/translation/CombineGloballyTest.java  | 101 ---
 .../spark/translation/CombinePerKeyTest.java    |  78 ---
 .../spark/translation/DoFnOutputTest.java       |  65 --
 .../translation/MultiOutputWordCountTest.java   | 174 -----
 .../spark/translation/SerializationTest.java    | 199 ------
 .../spark/translation/SideEffectsTest.java      |  20 +-
 .../translation/SparkPipelineOptionsTest.java   |  42 --
 .../translation/TransformTranslatorTest.java    | 104 ---
 .../translation/WindowedWordCountTest.java      | 116 ----
 .../streaming/EmptyStreamAssertionTest.java     |  10 +-
 .../ResumeFromCheckpointStreamingTest.java      |   2 +-
 .../streaming/SimpleStreamingWordCountTest.java |   1 -
 .../streaming/utils/PAssertStreaming.java       |   6 +-
 sdks/java/build-tools/pom.xml                   |   2 +-
 .../src/main/resources/beam/checkstyle.xml      |   9 +-
 .../src/main/resources/beam/suppressions.xml    |  12 +-
 sdks/java/core/pom.xml                          |   2 +-
 .../main/java/org/apache/beam/sdk/Pipeline.java |  12 +-
 .../org/apache/beam/sdk/PipelineResult.java     |   4 +-
 .../beam/sdk/annotations/Experimental.java      |   4 +-
 .../sdk/coders/CannotProvideCoderException.java |   2 +-
 .../java/org/apache/beam/sdk/coders/Coder.java  |  26 +-
 .../apache/beam/sdk/coders/CoderFactory.java    |   4 +-
 .../apache/beam/sdk/coders/CoderProvider.java   |   2 +-
 .../apache/beam/sdk/coders/DelegateCoder.java   |   4 +-
 .../apache/beam/sdk/coders/NullableCoder.java   |   6 +-
 .../java/org/apache/beam/sdk/io/AvroIO.java     |   4 +
 .../java/org/apache/beam/sdk/io/AvroSource.java |  97 ++-
 .../sdk/io/BoundedReadFromUnboundedSource.java  |   9 +-
 .../apache/beam/sdk/io/CompressedSource.java    |   8 +-
 .../org/apache/beam/sdk/io/FileBasedSink.java   |  14 +-
 .../apache/beam/sdk/io/OffsetBasedSource.java   |   4 +-
 .../java/org/apache/beam/sdk/io/PubsubIO.java   |   5 +-
 .../main/java/org/apache/beam/sdk/io/Read.java  |   4 +-
 .../sdk/io/SerializableAvroCodecFactory.java    |   2 +-
 .../java/org/apache/beam/sdk/io/TextIO.java     |   8 +-
 .../main/java/org/apache/beam/sdk/io/Write.java |   6 +-
 .../org/apache/beam/sdk/options/GcpOptions.java |   8 +-
 .../org/apache/beam/sdk/options/GcsOptions.java |   4 +-
 .../beam/sdk/options/GoogleApiDebugOptions.java |   2 +-
 .../beam/sdk/options/PipelineOptions.java       |  12 +-
 .../sdk/options/PipelineOptionsFactory.java     | 196 ++++--
 .../sdk/options/ProxyInvocationHandler.java     | 178 ++---
 .../apache/beam/sdk/options/ValueProvider.java  |  25 +-
 .../sdk/runners/PipelineRunnerRegistrar.java    |   2 +-
 .../beam/sdk/testing/BigqueryMatcher.java       | 239 +++++++
 .../org/apache/beam/sdk/testing/PAssert.java    |   4 +-
 .../apache/beam/sdk/transforms/Aggregator.java  |  11 +-
 .../org/apache/beam/sdk/transforms/Combine.java |  82 ++-
 .../beam/sdk/transforms/CombineFnBase.java      |  17 +-
 .../apache/beam/sdk/transforms/CombineFns.java  |  33 +-
 .../beam/sdk/transforms/CombineWithContext.java |   3 +-
 .../sdk/transforms/DelegatingAggregator.java    | 125 ++++
 .../org/apache/beam/sdk/transforms/DoFn.java    | 135 +++-
 .../beam/sdk/transforms/DoFnAdapters.java       |   5 +-
 .../apache/beam/sdk/transforms/DoFnTester.java  |  18 +-
 .../beam/sdk/transforms/FlatMapElements.java    |   6 +-
 .../apache/beam/sdk/transforms/MapElements.java |   8 +-
 .../org/apache/beam/sdk/transforms/OldDoFn.java | 301 +++++++--
 .../org/apache/beam/sdk/transforms/ParDo.java   | 277 +++++---
 .../apache/beam/sdk/transforms/Partition.java   |   4 +-
 .../sdk/transforms/SerializableFunction.java    |   2 +-
 .../sdk/transforms/display/DisplayData.java     | 518 ++++++++++-----
 .../sdk/transforms/reflect/DoFnInvokers.java    |  21 +-
 .../sdk/transforms/reflect/DoFnSignature.java   | 280 +++++++-
 .../sdk/transforms/reflect/DoFnSignatures.java  | 259 +++++++-
 .../beam/sdk/transforms/windowing/AfterAll.java |   2 +-
 .../windowing/AfterDelayFromFirstElement.java   |  15 +
 .../sdk/transforms/windowing/AfterEach.java     |   2 +-
 .../sdk/transforms/windowing/AfterFirst.java    |   2 +-
 .../sdk/transforms/windowing/AfterPane.java     |   7 +
 .../AfterSynchronizedProcessingTime.java        |   6 +-
 .../transforms/windowing/AfterWatermark.java    |  12 +-
 .../transforms/windowing/DefaultTrigger.java    |   2 +-
 .../beam/sdk/transforms/windowing/Never.java    |   8 +-
 .../transforms/windowing/OrFinallyTrigger.java  |  17 +-
 .../beam/sdk/transforms/windowing/PaneInfo.java |   2 +-
 .../sdk/transforms/windowing/Repeatedly.java    |  10 +-
 .../beam/sdk/transforms/windowing/Trigger.java  |  15 +-
 .../beam/sdk/transforms/windowing/Window.java   |   2 +-
 .../beam/sdk/transforms/windowing/WindowFn.java |   2 +-
 ...AttemptAndTimeBoundedExponentialBackOff.java |   2 +-
 .../apache/beam/sdk/util/CredentialFactory.java |   2 +-
 .../apache/beam/sdk/util/FinishedTriggers.java  |   8 +-
 .../apache/beam/sdk/util/IOChannelFactory.java  |   2 +-
 .../org/apache/beam/sdk/util/PathValidator.java |   6 +-
 .../beam/sdk/util/PerKeyCombineFnRunner.java    |  26 +-
 .../sdk/util/ReifyTimestampAndWindowsDoFn.java  |  16 +-
 .../apache/beam/sdk/util/ReshuffleTrigger.java  |   2 +-
 .../org/apache/beam/sdk/util/StringUtils.java   |   2 +-
 .../java/org/apache/beam/sdk/util/Timer.java    |  56 ++
 .../apache/beam/sdk/util/TimerInternals.java    |   4 +-
 .../org/apache/beam/sdk/util/TimerSpec.java     |  30 +
 .../org/apache/beam/sdk/util/TimerSpecs.java    |  41 ++
 .../java/org/apache/beam/sdk/util/Timers.java   |  10 +-
 .../apache/beam/sdk/util/common/Reiterable.java |   2 +-
 .../apache/beam/sdk/util/common/Reiterator.java |   2 +-
 .../CopyOnAccessInMemoryStateInternals.java     |   2 +-
 .../beam/sdk/util/state/ReadableState.java      |  10 +-
 .../beam/sdk/util/state/StateContext.java       |   6 +-
 .../apache/beam/sdk/values/PCollectionView.java |   8 +-
 .../java/org/apache/beam/sdk/values/PInput.java |   6 +-
 .../org/apache/beam/sdk/values/POutput.java     |   8 +-
 .../java/org/apache/beam/sdk/values/PValue.java |   4 +-
 .../apache/beam/sdk/coders/AvroCoderTest.java   |   4 +-
 .../beam/sdk/coders/NullableCoderTest.java      |  11 +
 .../java/org/apache/beam/sdk/io/AvroIOTest.java |  12 +-
 .../org/apache/beam/sdk/io/AvroSourceTest.java  |  43 ++
 .../io/BoundedReadFromUnboundedSourceTest.java  |   4 +-
 .../beam/sdk/io/CompressedSourceTest.java       |   4 +-
 .../beam/sdk/io/OffsetBasedSourceTest.java      |  30 +-
 .../java/org/apache/beam/sdk/io/ReadTest.java   |  10 +-
 .../io/SerializableAvroCodecFactoryTest.java    |   8 +-
 .../java/org/apache/beam/sdk/io/TextIOTest.java |  25 +
 .../java/org/apache/beam/sdk/io/WriteTest.java  |   6 +-
 .../sdk/options/PipelineOptionsFactoryTest.java |  69 +-
 .../beam/sdk/options/PipelineOptionsTest.java   |  49 +-
 .../options/PipelineOptionsValidatorTest.java   |  18 +-
 .../sdk/options/ProxyInvocationHandlerTest.java |  76 ++-
 .../beam/sdk/options/ValueProviderTest.java     |  13 +-
 .../beam/sdk/testing/BigqueryMatcherTest.java   | 176 +++++
 .../beam/sdk/transforms/CombineFnsTest.java     |   7 +-
 .../apache/beam/sdk/transforms/CombineTest.java |   4 +-
 .../DoFnDelegatingAggregatorTest.java           |   5 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   |  24 +-
 .../transforms/display/DisplayDataMatchers.java | 141 ++--
 .../display/DisplayDataMatchersTest.java        |  67 +-
 .../sdk/transforms/display/DisplayDataTest.java | 367 +++++++----
 .../DoFnSignaturesSplittableDoFnTest.java       |   6 +-
 .../transforms/reflect/DoFnSignaturesTest.java  | 645 ++++++++++++++++++-
 .../reflect/DoFnSignaturesTestUtils.java        |   5 +-
 .../sdk/transforms/windowing/WindowTest.java    |   4 +-
 .../apache/beam/sdk/util/ApiSurfaceTest.java    |  24 +-
 .../beam/sdk/util/ExecutableTriggerTest.java    | 127 ++++
 .../sdk/util/FinishedTriggersBitSetTest.java    |  55 ++
 .../sdk/util/FinishedTriggersProperties.java    | 110 ++++
 .../beam/sdk/util/FinishedTriggersSetTest.java  |  60 ++
 .../beam/sdk/util/ReshuffleTriggerTest.java     |  67 ++
 sdks/java/extensions/join-library/pom.xml       |   2 +-
 sdks/java/extensions/pom.xml                    |   2 +-
 sdks/java/io/google-cloud-platform/pom.xml      |   2 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    |  12 +-
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java    |   2 +-
 .../beam/sdk/io/gcp/datastore/DatastoreV1.java  |   2 +-
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     |  39 +-
 sdks/java/io/hdfs/pom.xml                       |   2 +-
 sdks/java/io/jdbc/pom.xml                       |   2 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java     |  70 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java |  39 +-
 sdks/java/io/jms/pom.xml                        |   2 +-
 sdks/java/io/kafka/pom.xml                      |   2 +-
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java   |  14 +-
 .../apache/beam/sdk/io/kafka/KafkaIOTest.java   |  26 +-
 sdks/java/io/kinesis/pom.xml                    |   2 +-
 sdks/java/io/mongodb/pom.xml                    |   2 +-
 sdks/java/io/pom.xml                            |   2 +-
 sdks/java/java8tests/pom.xml                    |   2 +-
 .../PipelineOptionsFactoryJava8Test.java        |   8 +-
 sdks/java/maven-archetypes/examples/pom.xml     |   2 +-
 .../main/resources/archetype-resources/pom.xml  |   8 +-
 .../src/main/java/DebuggingWordCount.java       |   2 +-
 .../src/main/java/WindowedWordCount.java        |   2 +-
 .../src/main/java/WordCount.java                |   2 +-
 sdks/java/maven-archetypes/pom.xml              |   2 +-
 sdks/java/maven-archetypes/starter/pom.xml      |   2 +-
 .../main/resources/archetype-resources/pom.xml  |   2 +-
 .../resources/projects/basic/reference/pom.xml  |   2 +-
 sdks/java/microbenchmarks/pom.xml               |   2 +-
 sdks/java/pom.xml                               |   2 +-
 sdks/pom.xml                                    |   2 +-
 298 files changed, 8342 insertions(+), 4136 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6a971d6b/runners/pom.xml
----------------------------------------------------------------------
diff --cc runners/pom.xml
index ff800d1,b2b613f..203b951
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@@ -36,8 -36,8 +36,9 @@@
      <module>core-java</module>
      <module>direct-java</module>
      <module>flink</module>
+     <module>google-cloud-dataflow-java</module>
      <module>spark</module>
 +    <module>apex</module>
    </modules>
  
    <profiles>


Mime
View raw message