beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [48/50] [abbrv] beam git commit: Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
Date Fri, 21 Apr 2017 17:53:32 GMT
Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner

# Conflicts:
#	.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
#	runners/pom.xml


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/46c41fce
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/46c41fce
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/46c41fce

Branch: refs/heads/gearpump-runner
Commit: 46c41fce0e130bdd718c0d3ce7c31a564524bae6
Parents: ebbb613 104f982
Author: manuzhang <owenzhang1990@gmail.com>
Authored: Thu Apr 20 20:59:22 2017 +0800
Committer: manuzhang <owenzhang1990@gmail.com>
Committed: Thu Apr 20 20:59:22 2017 +0800

----------------------------------------------------------------------
 .github/PULL_REQUEST_TEMPLATE.md                |    2 +-
 .gitignore                                      |    3 +
 .jenkins/common_job_properties.groovy           |  223 --
 ...job_beam_PostCommit_Java_MavenInstall.groovy |   42 -
 ...ostCommit_Java_RunnableOnService_Apex.groovy |   47 -
 ...ommit_Java_RunnableOnService_Dataflow.groovy |   45 -
 ...stCommit_Java_RunnableOnService_Flink.groovy |   44 -
 ...ommit_Java_RunnableOnService_Gearpump.groovy |   49 -
 ...stCommit_Java_RunnableOnService_Spark.groovy |   44 -
 .../job_beam_PostCommit_Python_Verify.groovy    |   43 -
 .../job_beam_PreCommit_Java_MavenInstall.groovy |   42 -
 .../job_beam_PreCommit_Website_Stage.groovy     |   80 -
 .jenkins/job_beam_PreCommit_Website_Test.groovy |   65 -
 .../job_beam_Release_NightlySnapshot.groovy     |   45 -
 .jenkins/job_seed.groovy                        |   53 -
 .../jenkins/common_job_properties.groovy        |  261 ++
 .../job_beam_PerformanceTests_Dataflow.groovy   |   43 +
 .../job_beam_PerformanceTests_JDBC.groovy       |   60 +
 .../job_beam_PerformanceTests_Spark.groovy      |   44 +
 ...job_beam_PostCommit_Java_MavenInstall.groovy |   42 +
 ..._PostCommit_Java_ValidatesRunner_Apex.groovy |   48 +
 ...tCommit_Java_ValidatesRunner_Dataflow.groovy |   45 +
 ...PostCommit_Java_ValidatesRunner_Flink.groovy |   43 +
 ...tCommit_Java_ValidatesRunner_Gearpump.groovy |   49 +
 ...PostCommit_Java_ValidatesRunner_Spark.groovy |   44 +
 .../job_beam_PostCommit_Python_Verify.groovy    |   55 +
 .../job_beam_PreCommit_Java_MavenInstall.groovy |   42 +
 .../job_beam_PreCommit_Website_Stage.groovy     |   80 +
 .../job_beam_PreCommit_Website_Test.groovy      |   65 +
 .../job_beam_Release_NightlySnapshot.groovy     |   45 +
 .test-infra/jenkins/job_seed.groovy             |   53 +
 .../cassandra-service-for-local-dev.yaml        |   28 +
 .../cassandra-svc-statefulset.yaml              |  114 +
 .../LargeITCluster/cassandra-svc-temp.yaml      |   74 +
 .../cassandra/LargeITCluster/data-load.sh       |  122 +
 .../cassandra/LargeITCluster/show_health.sh     |   47 +
 .../cassandra/LargeITCluster/start-up.sh        |   22 +
 .../cassandra/LargeITCluster/teardown.sh        |   25 +
 .../cassandra-service-for-local-dev.yaml        |   30 +
 .../SmallITCluster/cassandra-svc-rc.yaml        |   74 +
 .../cassandra/SmallITCluster/data-load.sh       |   86 +
 .../cassandra/SmallITCluster/show_health.sh     |   47 +
 .../cassandra/SmallITCluster/start-up.sh        |   23 +
 .../cassandra/SmallITCluster/teardown.sh        |   22 +
 .../kubernetes/cassandra/data-load-setup.sh     |   29 +
 .../elasticsearch-service-for-local-dev.yaml    |   33 +
 .../es-services-deployments.yaml                |  258 ++
 .../LargeProductionCluster/start-up.sh          |   22 +
 .../LargeProductionCluster/teardown.sh          |   21 +
 .../elasticsearch-service-for-local-dev.yaml    |   34 +
 .../SmallITCluster/elasticsearch-svc-rc.yaml    |   96 +
 .../elasticsearch/SmallITCluster/start-up.sh    |   23 +
 .../elasticsearch/SmallITCluster/teardown.sh    |   21 +
 .../kubernetes/elasticsearch/data-load-setup.sh |   26 +
 .../kubernetes/elasticsearch/data-load.sh       |   33 +
 .../kubernetes/elasticsearch/es_test_data.py    |  299 ++
 .../kubernetes/elasticsearch/show-health.sh     |   33 +
 .../postgres/postgres-service-for-local-dev.yml |   28 +
 .test-infra/kubernetes/postgres/postgres.yml    |   56 +
 .test-infra/travis/README.md                    |   23 +
 .test-infra/travis/settings.xml                 |   33 +
 .test-infra/travis/test_wordcount.sh            |  125 +
 .travis.yml                                     |    7 +-
 .travis/README.md                               |   23 -
 .travis/settings.xml                            |   33 -
 .travis/test_wordcount.sh                       |  125 -
 README.md                                       |    3 +
 examples/java/README.md                         |   61 +-
 examples/java/pom.xml                           |   32 +-
 .../beam/examples/DebuggingWordCount.java       |    2 +-
 .../apache/beam/examples/WindowedWordCount.java |   34 +-
 .../examples/common/WriteOneFilePerWindow.java  |   91 +
 .../examples/common/WriteWindowedFilesDoFn.java |   77 -
 .../beam/examples/complete/AutoComplete.java    |    2 +-
 .../examples/complete/StreamingWordExtract.java |    2 +-
 .../apache/beam/examples/complete/TfIdf.java    |    3 +-
 .../examples/complete/TrafficMaxLaneFlow.java   |    2 +-
 .../beam/examples/complete/TrafficRoutes.java   |    2 +-
 .../examples/cookbook/BigQueryTornadoes.java    |   12 +-
 .../cookbook/CombinePerKeyExamples.java         |    4 +-
 .../beam/examples/cookbook/FilterExamples.java  |    7 +-
 .../beam/examples/cookbook/JoinExamples.java    |    4 +-
 .../examples/cookbook/MaxPerKeyExamples.java    |    4 +-
 .../org/apache/beam/examples/cookbook/README.md |    2 +-
 .../beam/examples/cookbook/TriggerExample.java  |    4 +-
 .../beam/examples/WindowedWordCountIT.java      |   41 +-
 .../org/apache/beam/examples/WordCountTest.java |    4 +-
 .../beam/examples/complete/TfIdfTest.java       |    4 +-
 .../complete/TopWikipediaSessionsTest.java      |    4 +-
 .../examples/cookbook/DistinctExampleTest.java  |    6 +-
 .../examples/cookbook/JoinExamplesTest.java     |    4 +-
 .../examples/cookbook/TriggerExampleTest.java   |    4 +-
 examples/java8/pom.xml                          |   41 +-
 .../beam/examples/MinimalWordCountJava8.java    |    9 +-
 .../beam/examples/complete/game/GameStats.java  |   16 +-
 .../examples/complete/game/LeaderBoard.java     |    2 +-
 .../beam/examples/complete/game/README.md       |    6 +-
 .../beam/examples/complete/game/UserScore.java  |    5 +-
 .../complete/game/injector/Injector.java        |    3 +-
 .../complete/game/utils/WriteToBigQuery.java    |   13 +-
 .../game/utils/WriteWindowedToBigQuery.java     |    9 +-
 .../examples/MinimalWordCountJava8Test.java     |    9 +-
 .../examples/complete/game/GameStatsTest.java   |    6 +-
 .../complete/game/HourlyTeamScoreTest.java      |   11 +-
 .../examples/complete/game/UserScoreTest.java   |   14 +-
 pom.xml                                         |  161 +-
 runners/apex/pom.xml                            |   12 +-
 .../apache/beam/runners/apex/ApexRunner.java    |   81 +-
 .../beam/runners/apex/ApexYarnLauncher.java     |  111 +-
 .../translation/ApexPipelineTranslator.java     |    5 +-
 .../FlattenPCollectionTranslator.java           |   13 +-
 .../apex/translation/GroupByKeyTranslator.java  |    4 +-
 .../translation/ParDoBoundMultiTranslator.java  |  185 -
 .../apex/translation/ParDoBoundTranslator.java  |   95 -
 .../apex/translation/ParDoTranslator.java       |  185 +
 .../apex/translation/TranslationContext.java    |   22 +-
 .../translation/WindowAssignTranslator.java     |   58 +-
 .../operators/ApexGroupByKeyOperator.java       |  273 +-
 .../operators/ApexParDoOperator.java            |  238 +-
 .../operators/ApexProcessFnOperator.java        |  184 +
 .../translation/utils/ApexStateInternals.java   |  101 +-
 .../apex/translation/utils/NoOpStepContext.java |    2 +-
 .../utils/SerializablePipelineOptions.java      |   13 +-
 .../translation/utils/StateInternalsProxy.java  |   67 +
 .../apex/translation/utils/ValuesSource.java    |    2 +-
 .../apex/examples/UnboundedTextSource.java      |    2 +-
 .../translation/ApexGroupByKeyOperatorTest.java |    2 +-
 .../FlattenPCollectionTranslatorTest.java       |    3 +-
 .../translation/GroupByKeyTranslatorTest.java   |    2 +-
 .../translation/ParDoBoundTranslatorTest.java   |  344 --
 .../apex/translation/ParDoTranslatorTest.java   |  345 ++
 .../utils/ApexStateInternalsTest.java           |   43 +-
 .../translation/utils/CollectionSource.java     |    2 +-
 runners/core-construction-java/pom.xml          |   97 +-
 .../beam/runners/core/construction/Coders.java  |  174 +
 .../DeduplicatedFlattenFactory.java             |  120 +
 .../EmptyFlattenAsCreateFactory.java            |   34 +-
 .../core/construction/ForwardingPTransform.java |   62 +
 .../runners/core/construction/PCollections.java |   97 +
 .../core/construction/PTransformMatchers.java   |  135 +-
 .../construction/PTransformReplacements.java    |   69 +
 .../runners/core/construction/PTransforms.java  |  107 +
 .../core/construction/PrimitiveCreate.java      |   18 +-
 .../core/construction/ReplacementOutputs.java   |   63 +-
 .../core/construction/SdkComponents.java        |  195 ++
 .../SingleInputOutputOverrideFactory.java       |   14 +-
 .../runners/core/construction/Triggers.java     |  336 ++
 .../UnboundedReadFromBoundedSource.java         |  543 +++
 .../core/construction/UnconsumedReads.java      |   72 +
 .../UnsupportedOverrideFactory.java             |   20 +-
 .../core/construction/WindowingStrategies.java  |  245 ++
 .../runners/core/construction/CodersTest.java   |  163 +
 .../DeduplicatedFlattenFactoryTest.java         |  104 +
 .../EmptyFlattenAsCreateFactoryTest.java        |  122 +
 .../construction/ForwardingPTransformTest.java  |  111 +
 .../core/construction/PCollectionsTest.java     |  188 +
 .../construction/PTransformMatchersTest.java    |  198 +-
 .../PTransformReplacementsTest.java             |  131 +
 .../core/construction/PTransformsTest.java      |  189 +
 .../construction/ReplacementOutputsTest.java    |  109 +-
 .../core/construction/SdkComponentsTest.java    |  223 ++
 .../SingleInputOutputOverrideFactoryTest.java   |   37 +-
 .../runners/core/construction/TriggersTest.java |  111 +
 .../UnboundedReadFromBoundedSourceTest.java     |  373 ++
 .../core/construction/UnconsumedReadsTest.java  |  105 +
 .../UnsupportedOverrideFactoryTest.java         |   16 +-
 .../construction/WindowingStrategiesTest.java   |  110 +
 runners/core-java/pom.xml                       |   58 +-
 .../beam/runners/core/BaseExecutionContext.java |   13 +-
 .../apache/beam/runners/core/DoFnAdapters.java  |   21 +-
 .../apache/beam/runners/core/DoFnRunners.java   |   40 +-
 .../beam/runners/core/ExecutionContext.java     |   13 +-
 .../GroupAlsoByWindowViaOutputBufferDoFn.java   |    2 +-
 .../core/GroupAlsoByWindowViaWindowSetDoFn.java |    2 +-
 .../GroupAlsoByWindowViaWindowSetNewDoFn.java   |    8 +-
 .../runners/core/InMemoryStateInternals.java    |  120 +-
 .../apache/beam/runners/core/NonEmptyPanes.java |    4 +-
 .../org/apache/beam/runners/core/OldDoFn.java   |   49 +-
 ...eBoundedSplittableProcessElementInvoker.java |  136 +-
 .../beam/runners/core/OutputWindowedValue.java  |   10 +-
 .../beam/runners/core/ProcessFnRunner.java      |  127 +
 .../core/PushbackSideInputDoFnRunner.java       |  106 +-
 .../runners/core/ReduceFnContextFactory.java    |    3 +-
 .../beam/runners/core/SideInputHandler.java     |   18 +-
 .../beam/runners/core/SimpleDoFnRunner.java     |   76 +-
 .../beam/runners/core/SimpleOldDoFnRunner.java  |   65 +-
 .../core/SimplePushbackSideInputDoFnRunner.java |  115 +
 .../beam/runners/core/SplittableParDo.java      |  154 +-
 .../core/SplittableProcessElementInvoker.java   |   22 +-
 .../apache/beam/runners/core/StateMerging.java  |   16 +-
 .../org/apache/beam/runners/core/StateTag.java  |   14 +-
 .../org/apache/beam/runners/core/StateTags.java |   32 +-
 .../beam/runners/core/StatefulDoFnRunner.java   |   96 +
 .../beam/runners/core/SystemReduceFn.java       |    8 +-
 .../core/UnboundedReadFromBoundedSource.java    |  536 ---
 .../beam/runners/core/WindowingInternals.java   |    8 +-
 .../core/WindowingInternalsAdapters.java        |    8 +-
 .../AfterDelayFromFirstElementStateMachine.java |   10 +-
 .../core/triggers/AfterPaneStateMachine.java    |    4 +-
 .../triggers/AfterWatermarkStateMachine.java    |   14 +-
 .../core/GroupAlsoByWindowsProperties.java      |   10 +-
 .../core/InMemoryStateInternalsTest.java        |  105 +-
 .../apache/beam/runners/core/NoOpOldDoFn.java   |    4 +-
 .../apache/beam/runners/core/OldDoFnTest.java   |    4 +-
 ...ndedSplittableProcessElementInvokerTest.java |   27 +-
 .../core/PushbackSideInputDoFnRunnerTest.java   |  282 --
 .../beam/runners/core/ReduceFnRunnerTest.java   |   10 +-
 .../beam/runners/core/ReduceFnTester.java       |   10 +-
 .../beam/runners/core/SideInputHandlerTest.java |   22 +-
 .../beam/runners/core/SimpleDoFnRunnerTest.java |  145 +
 .../runners/core/SimpleOldDoFnRunnerTest.java   |    4 +-
 .../SimplePushbackSideInputDoFnRunnerTest.java  |  282 ++
 .../beam/runners/core/SplittableParDoTest.java  |  332 +-
 .../runners/core/StatefulDoFnRunnerTest.java    |  113 +-
 .../UnboundedReadFromBoundedSourceTest.java     |  373 --
 runners/direct-java/pom.xml                     |   55 +-
 .../direct/BoundedReadEvaluatorFactory.java     |    4 +-
 .../CopyOnAccessInMemoryStateInternals.java     |   46 +-
 ...ectGBKIntoKeyedWorkItemsOverrideFactory.java |   16 +-
 .../beam/runners/direct/DirectGraphVisitor.java |    5 +-
 .../beam/runners/direct/DirectGroupByKey.java   |    1 +
 .../direct/DirectGroupByKeyOverrideFactory.java |   14 +-
 .../beam/runners/direct/DirectMetrics.java      |  129 +-
 .../beam/runners/direct/DirectOptions.java      |   11 -
 .../beam/runners/direct/DirectRunner.java       |  132 +-
 ...ecycleManagerRemovingTransformEvaluator.java |    6 +-
 .../beam/runners/direct/EvaluationContext.java  |   34 +-
 .../direct/ExecutorServiceParallelExecutor.java |  141 +-
 .../runners/direct/FlattenEvaluatorFactory.java |    2 +-
 .../runners/direct/ForwardingPTransform.java    |   62 -
 .../GroupAlsoByWindowEvaluatorFactory.java      |   12 +-
 .../direct/GroupByKeyOnlyEvaluatorFactory.java  |    4 +-
 .../direct/KeyedPValueTrackingVisitor.java      |   18 +-
 .../beam/runners/direct/ModelEnforcement.java   |   13 +-
 .../beam/runners/direct/ParDoEvaluator.java     |  129 +-
 .../runners/direct/ParDoEvaluatorFactory.java   |   39 +-
 .../direct/ParDoMultiOverrideFactory.java       |   53 +-
 .../ParDoSingleViaMultiOverrideFactory.java     |   70 -
 .../beam/runners/direct/PipelineExecutor.java   |   23 +-
 ...littableProcessElementsEvaluatorFactory.java |  108 +-
 .../direct/StatefulParDoEvaluatorFactory.java   |   31 +-
 .../direct/TestStreamEvaluatorFactory.java      |   20 +-
 .../direct/TransformEvaluatorFactory.java       |   10 +-
 .../direct/TransformEvaluatorRegistry.java      |    7 +-
 .../direct/TransformExecutorService.java        |    6 +
 .../direct/TransformExecutorServices.java       |   57 +-
 .../direct/UnboundedReadEvaluatorFactory.java   |   39 +-
 .../runners/direct/ViewEvaluatorFactory.java    |   82 +-
 .../runners/direct/ViewOverrideFactory.java     |  114 +
 .../beam/runners/direct/WatermarkManager.java   |   19 +-
 .../runners/direct/WindowEvaluatorFactory.java  |    2 +-
 .../direct/WriteWithShardingFactory.java        |   26 +-
 .../direct/BoundedReadEvaluatorFactoryTest.java |    4 +-
 .../CopyOnAccessInMemoryStateInternalsTest.java |   34 +-
 .../runners/direct/DirectGraphVisitorTest.java  |    7 +-
 .../DirectGroupByKeyOverrideFactoryTest.java    |   12 +-
 .../beam/runners/direct/DirectMetricsTest.java  |  124 +-
 .../beam/runners/direct/DirectRunnerTest.java   |   89 +-
 ...leManagerRemovingTransformEvaluatorTest.java |    8 +-
 .../runners/direct/EvaluationContextTest.java   |   57 +-
 .../direct/ForwardingPTransformTest.java        |  112 -
 .../beam/runners/direct/ParDoEvaluatorTest.java |   15 +-
 .../direct/ParDoMultiOverrideFactoryTest.java   |   45 -
 .../ParDoSingleViaMultiOverrideFactoryTest.java |   46 -
 .../runners/direct/SideInputContainerTest.java  |   19 +-
 .../StatefulParDoEvaluatorFactoryTest.java      |    5 +-
 .../direct/TestStreamEvaluatorFactoryTest.java  |   11 -
 .../direct/TransformExecutorServicesTest.java   |   48 +
 .../UnboundedReadEvaluatorFactoryTest.java      |   88 +-
 .../direct/ViewEvaluatorFactoryTest.java        |   14 +-
 .../runners/direct/ViewOverrideFactoryTest.java |  138 +
 .../direct/WindowEvaluatorFactoryTest.java      |   12 +-
 .../direct/WriteWithShardingFactoryTest.java    |   28 +-
 runners/flink/examples/pom.xml                  |  126 -
 .../beam/runners/flink/examples/TFIDF.java      |  456 ---
 .../beam/runners/flink/examples/WordCount.java  |  129 -
 .../runners/flink/examples/package-info.java    |   22 -
 .../flink/examples/streaming/AutoComplete.java  |  400 ---
 .../flink/examples/streaming/JoinExamples.java  |  154 -
 .../examples/streaming/KafkaIOExamples.java     |  338 --
 .../KafkaWindowedWordCountExample.java          |  164 -
 .../examples/streaming/WindowedWordCount.java   |  141 -
 .../flink/examples/streaming/package-info.java  |   22 -
 runners/flink/pom.xml                           |  320 +-
 runners/flink/runner/pom.xml                    |  310 --
 .../flink/DefaultParallelismFactory.java        |   39 -
 .../flink/FlinkBatchPipelineTranslator.java     |  139 -
 .../flink/FlinkBatchTransformTranslators.java   |  797 -----
 .../flink/FlinkBatchTranslationContext.java     |  154 -
 .../flink/FlinkDetachedRunnerResult.java        |   76 -
 .../FlinkPipelineExecutionEnvironment.java      |  241 --
 .../runners/flink/FlinkPipelineOptions.java     |  101 -
 .../runners/flink/FlinkPipelineTranslator.java  |   53 -
 .../apache/beam/runners/flink/FlinkRunner.java  |  250 --
 .../runners/flink/FlinkRunnerRegistrar.java     |   62 -
 .../beam/runners/flink/FlinkRunnerResult.java   |   98 -
 .../flink/FlinkStreamingPipelineTranslator.java |  230 --
 .../FlinkStreamingTransformTranslators.java     | 1043 ------
 .../flink/FlinkStreamingTranslationContext.java |  130 -
 .../flink/FlinkStreamingViewOverrides.java      |  372 --
 .../flink/PipelineTranslationOptimizer.java     |   72 -
 .../beam/runners/flink/TestFlinkRunner.java     |   97 -
 .../beam/runners/flink/TranslationMode.java     |   31 -
 .../apache/beam/runners/flink/package-info.java |   22 -
 .../functions/FlinkAggregatorFactory.java       |   53 -
 .../functions/FlinkAssignContext.java           |   63 -
 .../functions/FlinkAssignWindows.java           |   49 -
 .../functions/FlinkDoFnFunction.java            |  161 -
 .../FlinkMergingNonShuffleReduceFunction.java   |  228 --
 .../FlinkMergingPartialReduceFunction.java      |  201 --
 .../functions/FlinkMergingReduceFunction.java   |  199 --
 .../FlinkMultiOutputPruningFunction.java        |   50 -
 .../functions/FlinkNoOpStepContext.java         |   73 -
 .../functions/FlinkPartialReduceFunction.java   |  172 -
 .../functions/FlinkReduceFunction.java          |  173 -
 .../functions/FlinkSideInputReader.java         |   80 -
 .../functions/FlinkStatefulDoFnFunction.java    |  198 --
 .../functions/SideInputInitializer.java         |   73 -
 .../translation/functions/package-info.java     |   22 -
 .../runners/flink/translation/package-info.java |   22 -
 .../translation/types/CoderTypeInformation.java |  120 -
 .../translation/types/CoderTypeSerializer.java  |  132 -
 .../types/EncodedValueComparator.java           |  195 --
 .../types/EncodedValueSerializer.java           |  113 -
 .../types/EncodedValueTypeInformation.java      |  107 -
 .../flink/translation/types/FlinkCoder.java     |   63 -
 .../types/InspectableByteArrayOutputStream.java |   34 -
 .../flink/translation/types/KvKeySelector.java  |   50 -
 .../flink/translation/types/package-info.java   |   22 -
 .../utils/SerializedPipelineOptions.java        |   65 -
 .../flink/translation/utils/package-info.java   |   22 -
 .../wrappers/DataInputViewWrapper.java          |   58 -
 .../wrappers/DataOutputViewWrapper.java         |   51 -
 .../SerializableFnAggregatorWrapper.java        |   98 -
 .../translation/wrappers/SourceInputFormat.java |  149 -
 .../translation/wrappers/SourceInputSplit.java  |   52 -
 .../translation/wrappers/package-info.java      |   22 -
 .../wrappers/streaming/DoFnOperator.java        |  861 -----
 .../streaming/KvToByteBufferKeySelector.java    |   56 -
 .../streaming/SingletonKeyedWorkItem.java       |   58 -
 .../streaming/SingletonKeyedWorkItemCoder.java  |  128 -
 .../wrappers/streaming/WindowDoFnOperator.java  |  120 -
 .../wrappers/streaming/WorkItemKeySelector.java |   56 -
 .../streaming/io/BoundedSourceWrapper.java      |  218 --
 .../streaming/io/UnboundedFlinkSink.java        |  182 -
 .../streaming/io/UnboundedFlinkSource.java      |  120 -
 .../streaming/io/UnboundedSocketSource.java     |  249 --
 .../streaming/io/UnboundedSourceWrapper.java    |  476 ---
 .../wrappers/streaming/io/package-info.java     |   22 -
 .../wrappers/streaming/package-info.java        |   22 -
 .../state/FlinkBroadcastStateInternals.java     |  865 -----
 .../state/FlinkKeyGroupStateInternals.java      |  487 ---
 .../state/FlinkSplitStateInternals.java         |  260 --
 .../streaming/state/FlinkStateInternals.java    | 1053 ------
 .../state/KeyGroupCheckpointedOperator.java     |   35 -
 .../state/KeyGroupRestoringOperator.java        |   32 -
 .../wrappers/streaming/state/package-info.java  |   22 -
 .../runner/src/main/resources/log4j.properties  |   23 -
 .../flink/EncodedValueComparatorTest.java       |   70 -
 .../runners/flink/FlinkRunnerRegistrarTest.java |   48 -
 .../beam/runners/flink/FlinkTestPipeline.java   |   72 -
 .../beam/runners/flink/PipelineOptionsTest.java |  184 -
 .../beam/runners/flink/ReadSourceITCase.java    |   85 -
 .../flink/ReadSourceStreamingITCase.java        |   74 -
 .../beam/runners/flink/WriteSinkITCase.java     |  171 -
 .../flink/streaming/DoFnOperatorTest.java       |  594 ----
 .../FlinkBroadcastStateInternalsTest.java       |  245 --
 .../FlinkKeyGroupStateInternalsTest.java        |  262 --
 .../streaming/FlinkSplitStateInternalsTest.java |  101 -
 .../streaming/FlinkStateInternalsTest.java      |  395 ---
 .../flink/streaming/GroupByNullKeyTest.java     |  124 -
 .../flink/streaming/TestCountingSource.java     |  254 --
 .../streaming/TopWikipediaSessionsITCase.java   |  133 -
 .../streaming/UnboundedSourceWrapperTest.java   |  464 ---
 .../runners/flink/streaming/package-info.java   |   22 -
 .../src/test/resources/log4j-test.properties    |   27 -
 .../flink/DefaultParallelismFactory.java        |   39 +
 .../flink/FlinkBatchPipelineTranslator.java     |  139 +
 .../flink/FlinkBatchTransformTranslators.java   |  723 ++++
 .../flink/FlinkBatchTranslationContext.java     |  153 +
 .../flink/FlinkDetachedRunnerResult.java        |   75 +
 .../FlinkPipelineExecutionEnvironment.java      |  241 ++
 .../runners/flink/FlinkPipelineOptions.java     |  101 +
 .../runners/flink/FlinkPipelineTranslator.java  |   53 +
 .../apache/beam/runners/flink/FlinkRunner.java  |  232 ++
 .../runners/flink/FlinkRunnerRegistrar.java     |   62 +
 .../beam/runners/flink/FlinkRunnerResult.java   |   98 +
 .../flink/FlinkStreamingPipelineTranslator.java |  276 ++
 .../FlinkStreamingTransformTranslators.java     | 1044 ++++++
 .../flink/FlinkStreamingTranslationContext.java |  130 +
 .../flink/FlinkStreamingViewOverrides.java      |  372 ++
 .../flink/PipelineTranslationOptimizer.java     |   72 +
 .../beam/runners/flink/TestFlinkRunner.java     |   84 +
 .../beam/runners/flink/TranslationMode.java     |   31 +
 .../apache/beam/runners/flink/package-info.java |   22 +
 .../functions/FlinkAggregatorFactory.java       |   53 +
 .../functions/FlinkAssignContext.java           |   63 +
 .../functions/FlinkAssignWindows.java           |   49 +
 .../functions/FlinkDoFnFunction.java            |  161 +
 .../FlinkMergingNonShuffleReduceFunction.java   |  228 ++
 .../FlinkMergingPartialReduceFunction.java      |  201 ++
 .../functions/FlinkMergingReduceFunction.java   |  199 ++
 .../FlinkMultiOutputPruningFunction.java        |   50 +
 .../functions/FlinkNoOpStepContext.java         |   73 +
 .../functions/FlinkPartialReduceFunction.java   |  172 +
 .../functions/FlinkReduceFunction.java          |  173 +
 .../functions/FlinkSideInputReader.java         |   80 +
 .../functions/FlinkStatefulDoFnFunction.java    |  198 ++
 .../functions/SideInputInitializer.java         |   73 +
 .../translation/functions/package-info.java     |   22 +
 .../runners/flink/translation/package-info.java |   22 +
 .../translation/types/CoderTypeInformation.java |  120 +
 .../translation/types/CoderTypeSerializer.java  |  132 +
 .../types/EncodedValueComparator.java           |  195 ++
 .../types/EncodedValueSerializer.java           |  113 +
 .../types/EncodedValueTypeInformation.java      |   98 +
 .../types/InspectableByteArrayOutputStream.java |   34 +
 .../flink/translation/types/KvKeySelector.java  |   50 +
 .../flink/translation/types/package-info.java   |   22 +
 .../utils/SerializedPipelineOptions.java        |   67 +
 .../flink/translation/utils/package-info.java   |   22 +
 .../wrappers/DataInputViewWrapper.java          |   58 +
 .../wrappers/DataOutputViewWrapper.java         |   51 +
 .../SerializableFnAggregatorWrapper.java        |   98 +
 .../translation/wrappers/SourceInputFormat.java |  150 +
 .../translation/wrappers/SourceInputSplit.java  |   52 +
 .../translation/wrappers/package-info.java      |   22 +
 .../wrappers/streaming/DoFnOperator.java        |  774 ++++
 .../streaming/KvToByteBufferKeySelector.java    |   56 +
 .../streaming/SingletonKeyedWorkItem.java       |   56 +
 .../streaming/SingletonKeyedWorkItemCoder.java  |  126 +
 .../streaming/SplittableDoFnOperator.java       |  150 +
 .../wrappers/streaming/WindowDoFnOperator.java  |  117 +
 .../wrappers/streaming/WorkItemKeySelector.java |   56 +
 .../streaming/io/BoundedSourceWrapper.java      |  218 ++
 .../streaming/io/UnboundedSocketSource.java     |  249 ++
 .../streaming/io/UnboundedSourceWrapper.java    |  476 +++
 .../wrappers/streaming/io/package-info.java     |   22 +
 .../wrappers/streaming/package-info.java        |   22 +
 .../state/FlinkBroadcastStateInternals.java     |  865 +++++
 .../state/FlinkKeyGroupStateInternals.java      |  487 +++
 .../state/FlinkSplitStateInternals.java         |  260 ++
 .../streaming/state/FlinkStateInternals.java    | 1053 ++++++
 .../state/KeyGroupCheckpointedOperator.java     |   35 +
 .../state/KeyGroupRestoringOperator.java        |   32 +
 .../wrappers/streaming/state/package-info.java  |   22 +
 .../flink/src/main/resources/log4j.properties   |   23 +
 .../flink/EncodedValueComparatorTest.java       |   70 +
 .../runners/flink/FlinkRunnerRegistrarTest.java |   48 +
 .../beam/runners/flink/FlinkTestPipeline.java   |   72 +
 .../beam/runners/flink/PipelineOptionsTest.java |  184 +
 .../beam/runners/flink/ReadSourceITCase.java    |   85 +
 .../flink/ReadSourceStreamingITCase.java        |   74 +
 .../beam/runners/flink/WriteSinkITCase.java     |  192 +
 .../flink/streaming/DoFnOperatorTest.java       |  600 ++++
 .../FlinkBroadcastStateInternalsTest.java       |  245 ++
 .../FlinkKeyGroupStateInternalsTest.java        |  262 ++
 .../streaming/FlinkSplitStateInternalsTest.java |  101 +
 .../streaming/FlinkStateInternalsTest.java      |  395 +++
 .../flink/streaming/GroupByNullKeyTest.java     |  124 +
 .../flink/streaming/TestCountingSource.java     |  254 ++
 .../streaming/TopWikipediaSessionsITCase.java   |  133 +
 .../streaming/UnboundedSourceWrapperTest.java   |  464 +++
 .../runners/flink/streaming/package-info.java   |   22 +
 .../src/test/resources/log4j-test.properties    |   27 +
 .../translators/ParDoBoundMultiTranslator.java  |  104 -
 .../translators/ParDoBoundTranslator.java       |   75 -
 .../translators/ParDoMultiOutputTranslator.java |  104 +
 .../ParDoSingleOutputTranslator.java            |   75 +
 runners/google-cloud-dataflow-java/pom.xml      |   88 +-
 .../beam/runners/dataflow/AssignWindows.java    |    4 +-
 .../dataflow/BatchStatefulParDoOverrides.java   |   78 +-
 .../runners/dataflow/BatchViewOverrides.java    |   85 +-
 .../runners/dataflow/CreateDataflowView.java    |   46 +
 .../beam/runners/dataflow/DataflowMetrics.java  |  225 ++
 .../runners/dataflow/DataflowPipelineJob.java   |   96 +-
 .../dataflow/DataflowPipelineTranslator.java    |   71 +-
 .../beam/runners/dataflow/DataflowRunner.java   |  387 +-
 .../runners/dataflow/DataflowRunnerInfo.java    |   38 +-
 .../DataflowUnboundedReadFromBoundedSource.java |  547 ---
 .../dataflow/PrimitiveParDoSingleFactory.java   |   85 +
 .../dataflow/ReshuffleOverrideFactory.java      |   90 +
 .../dataflow/StreamingViewOverrides.java        |   17 +-
 .../runners/dataflow/TransformTranslator.java   |    6 +-
 .../dataflow/internal/CustomSources.java        |    2 +-
 .../runners/dataflow/internal/IsmFormat.java    |    3 +-
 .../options/DataflowPipelineDebugOptions.java   |    4 +-
 .../DataflowPipelineWorkerPoolOptions.java      |    8 +-
 .../dataflow/testing/TestDataflowRunner.java    |   43 +-
 .../runners/dataflow/testing/package-info.java  |    2 +-
 .../runners/dataflow/util/MonitoringUtil.java   |    4 +
 .../beam/runners/dataflow/dataflow.properties   |    8 +-
 .../BatchStatefulParDoOverridesTest.java        |    2 +-
 .../dataflow/BatchViewOverridesTest.java        |    4 +-
 .../runners/dataflow/DataflowMetricsTest.java   |  236 ++
 .../dataflow/DataflowPipelineJobTest.java       |   50 +-
 .../DataflowPipelineTranslatorTest.java         |   85 +-
 .../dataflow/DataflowRunnerInfoTest.java        |   23 +-
 .../runners/dataflow/DataflowRunnerTest.java    |   98 +-
 ...aflowUnboundedReadFromBoundedSourceTest.java |   79 -
 .../PrimitiveParDoSingleFactoryTest.java        |  161 +
 .../testing/TestDataflowRunnerTest.java         |    3 +-
 .../dataflow/util/MonitoringUtilTest.java       |   31 +-
 runners/pom.xml                                 |   40 -
 runners/spark/pom.xml                           |   78 +-
 .../spark/SparkNativePipelineVisitor.java       |    1 -
 .../beam/runners/spark/SparkPipelineResult.java |   81 +-
 .../apache/beam/runners/spark/SparkRunner.java  |  195 +-
 .../beam/runners/spark/TestSparkRunner.java     |   45 +-
 .../spark/coders/StatelessJavaSerializer.java   |   11 +
 .../beam/runners/spark/io/MicrobatchSource.java |  118 +-
 .../beam/runners/spark/io/SourceDStream.java    |   63 +-
 .../apache/beam/runners/spark/io/SourceRDD.java |   74 +-
 .../runners/spark/io/SparkUnboundedSource.java  |  132 +-
 .../beam/runners/spark/io/hadoop/HadoopIO.java  |  216 --
 .../spark/io/hadoop/ShardNameBuilder.java       |  111 -
 .../spark/io/hadoop/ShardNameTemplateAware.java |   31 -
 .../io/hadoop/ShardNameTemplateHelper.java      |   63 -
 .../io/hadoop/TemplatedAvroKeyOutputFormat.java |   45 -
 .../TemplatedSequenceFileOutputFormat.java      |   45 -
 .../io/hadoop/TemplatedTextOutputFormat.java    |   45 -
 .../runners/spark/io/hadoop/package-info.java   |   22 -
 .../runners/spark/metrics/SparkBeamMetric.java  |    4 +
 .../spark/metrics/SparkMetricResults.java       |   67 +-
 .../spark/metrics/SparkMetricsContainer.java    |   31 +-
 .../SparkGroupAlsoByWindowViaWindowSet.java     |   48 +-
 .../spark/stateful/SparkStateInternals.java     |   30 +-
 .../spark/stateful/StateSpecFunctions.java      |   61 +-
 .../spark/translation/BoundedDataset.java       |    3 +-
 .../runners/spark/translation/DoFnFunction.java |  130 -
 .../spark/translation/EvaluationContext.java    |   72 +-
 .../translation/GroupCombineFunctions.java      |   53 +-
 .../spark/translation/MultiDoFnFunction.java    |    4 +-
 ...SparkGroupAlsoByWindowViaOutputBufferFn.java |   10 +-
 .../spark/translation/SparkProcessContext.java  |    2 +-
 .../spark/translation/SparkRuntimeContext.java  |    2 +
 .../spark/translation/TransformTranslator.java  |  370 +-
 .../spark/translation/TranslationUtils.java     |  245 +-
 .../SparkRunnerStreamingContextFactory.java     |   45 +-
 .../streaming/StreamingTransformTranslator.java |  158 +-
 .../spark/util/GlobalWatermarkHolder.java       |    2 +-
 .../spark/util/SparkSideInputReader.java        |    3 +-
 .../apache/beam/runners/spark/CacheTest.java    |   61 +
 .../runners/spark/SparkPipelineStateTest.java   |    2 +-
 .../runners/spark/UsesCheckpointRecovery.java   |   23 +
 .../io/hadoop/HadoopFileFormatPipelineTest.java |  121 -
 .../spark/io/hadoop/ShardNameBuilderTest.java   |   88 -
 .../spark/translation/StorageLevelTest.java     |    6 +-
 .../translation/streaming/CreateStreamTest.java |    4 +-
 .../ResumeFromCheckpointStreamingTest.java      |   17 +-
 .../streaming/StreamingSourceMetricsTest.java   |   71 +
 .../streaming/TrackStreamingSourcesTest.java    |    4 +-
 runners/spark/src/test/resources/test_text.txt  |    2 -
 sdks/common/fn-api/pom.xml                      |    5 -
 .../fn-api/src/main/proto/beam_fn_api.proto     |  174 +-
 .../src/main/proto/beam_runner_api.proto        |   60 +-
 sdks/java/build-tools/pom.xml                   |   31 +-
 .../src/main/resources/beam/beam-codestyle.xml  |  329 ++
 .../src/main/resources/beam/findbugs-filter.xml |   43 +-
 .../beam/codestyle/CodeStyleTestClass.java      |   45 +
 sdks/java/core/pom.xml                          |  155 +-
 .../beam/sdk/AggregatorPipelineExtractor.java   |    8 +-
 .../main/java/org/apache/beam/sdk/Pipeline.java |  123 +-
 .../java/org/apache/beam/sdk/io/AvroIO.java     |  170 +-
 .../java/org/apache/beam/sdk/io/AvroSource.java |    4 +-
 .../sdk/io/BoundedReadFromUnboundedSource.java  |    4 +-
 .../org/apache/beam/sdk/io/BoundedSource.java   |   20 +-
 .../apache/beam/sdk/io/CompressedSource.java    |   17 +-
 .../org/apache/beam/sdk/io/CountingInput.java   |    4 +
 .../org/apache/beam/sdk/io/CountingSource.java  |   10 +-
 .../org/apache/beam/sdk/io/FileBasedSink.java   |  429 ++-
 .../org/apache/beam/sdk/io/FileBasedSource.java |   38 +-
 .../org/apache/beam/sdk/io/FileSystems.java     |  302 +-
 .../org/apache/beam/sdk/io/LocalFileSystem.java |   57 +-
 .../org/apache/beam/sdk/io/LocalResourceId.java |    2 +-
 .../apache/beam/sdk/io/OffsetBasedSource.java   |    4 +-
 .../java/org/apache/beam/sdk/io/PubsubIO.java   | 1201 -------
 .../apache/beam/sdk/io/PubsubUnboundedSink.java |  500 ---
 .../beam/sdk/io/PubsubUnboundedSource.java      | 1435 --------
 .../main/java/org/apache/beam/sdk/io/Sink.java  |   57 +-
 .../java/org/apache/beam/sdk/io/TFRecordIO.java |  694 ++++
 .../java/org/apache/beam/sdk/io/TextIO.java     |  104 +-
 .../org/apache/beam/sdk/io/UnboundedSource.java |   19 +-
 .../main/java/org/apache/beam/sdk/io/Write.java |  377 +-
 .../main/java/org/apache/beam/sdk/io/XmlIO.java |  477 +++
 .../java/org/apache/beam/sdk/io/XmlSink.java    |  226 +-
 .../java/org/apache/beam/sdk/io/XmlSource.java  |  191 +-
 .../org/apache/beam/sdk/io/fs/MoveOptions.java  |   34 +
 .../org/apache/beam/sdk/io/fs/ResourceId.java   |    8 +
 .../org/apache/beam/sdk/io/package-info.java    |    2 +-
 .../apache/beam/sdk/io/range/ByteKeyRange.java  |    4 +-
 .../java/org/apache/beam/sdk/metrics/Gauge.java |   32 +
 .../org/apache/beam/sdk/metrics/GaugeCell.java  |   60 +
 .../org/apache/beam/sdk/metrics/GaugeData.java  |   81 +
 .../apache/beam/sdk/metrics/GaugeResult.java    |   61 +
 .../beam/sdk/metrics/MetricFiltering.java       |   99 +
 .../beam/sdk/metrics/MetricNameFilter.java      |    3 +-
 .../beam/sdk/metrics/MetricQueryResults.java    |    3 +
 .../apache/beam/sdk/metrics/MetricUpdates.java  |   11 +-
 .../org/apache/beam/sdk/metrics/Metrics.java    |   35 +
 .../beam/sdk/metrics/MetricsContainer.java      |   26 +-
 .../beam/sdk/options/BigQueryOptions.java       |   32 -
 .../options/CloudResourceManagerOptions.java    |   40 -
 .../DefaultPipelineOptionsRegistrar.java        |    5 -
 .../org/apache/beam/sdk/options/GcpOptions.java |  227 --
 .../org/apache/beam/sdk/options/GcsOptions.java |  158 -
 .../beam/sdk/options/GoogleApiDebugOptions.java |   87 -
 .../beam/sdk/options/PipelineOptions.java       |   16 +-
 .../apache/beam/sdk/options/PubsubOptions.java  |   36 -
 .../beam/sdk/options/StreamingOptions.java      |    7 +-
 .../beam/sdk/runners/PTransformOverride.java    |   44 +
 .../sdk/runners/PTransformOverrideFactory.java  |   31 +-
 .../apache/beam/sdk/runners/PipelineRunner.java |    7 +-
 .../beam/sdk/runners/TransformHierarchy.java    |   80 +-
 .../beam/sdk/testing/BigqueryMatcher.java       |  256 --
 .../apache/beam/sdk/testing/GatherAllPanes.java |    1 +
 .../org/apache/beam/sdk/testing/PAssert.java    |  220 +-
 .../apache/beam/sdk/testing/PaneExtractors.java |    1 +
 .../beam/sdk/testing/RunnableOnService.java     |   38 -
 .../beam/sdk/testing/SerializableMatcher.java   |    2 +-
 .../beam/sdk/testing/SerializableMatchers.java  |    4 +-
 .../beam/sdk/testing/SourceTestUtils.java       |    6 +-
 .../apache/beam/sdk/testing/StaticWindows.java  |   17 +-
 .../apache/beam/sdk/testing/StreamingIT.java    |    2 +-
 .../apache/beam/sdk/testing/TestPipeline.java   |   89 +-
 .../beam/sdk/testing/TestPipelineOptions.java   |    5 +
 ...esSplittableParDoWithWindowedSideInputs.java |   26 +
 .../beam/sdk/testing/ValidatesRunner.java       |   23 +
 .../beam/sdk/testing/ValueInSingleWindow.java   |  134 -
 .../beam/sdk/transforms/AppliedPTransform.java  |   24 +-
 .../org/apache/beam/sdk/transforms/Combine.java |   64 +-
 .../org/apache/beam/sdk/transforms/Create.java  |   86 +-
 .../org/apache/beam/sdk/transforms/DoFn.java    |  138 +-
 .../apache/beam/sdk/transforms/DoFnTester.java  |   51 +-
 .../beam/sdk/transforms/FlatMapElements.java    |  113 +-
 .../apache/beam/sdk/transforms/GroupByKey.java  |    6 +-
 .../beam/sdk/transforms/GroupIntoBatches.java   |  229 ++
 .../apache/beam/sdk/transforms/MapElements.java |   99 +-
 .../beam/sdk/transforms/Materialization.java    |   36 +
 .../beam/sdk/transforms/Materializations.java   |   53 +
 .../apache/beam/sdk/transforms/PTransform.java  |   16 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  495 +--
 .../apache/beam/sdk/transforms/Partition.java   |    6 +-
 .../org/apache/beam/sdk/transforms/Regex.java   |   14 +-
 .../org/apache/beam/sdk/transforms/Sample.java  |   11 +-
 .../org/apache/beam/sdk/transforms/View.java    |   81 +-
 .../org/apache/beam/sdk/transforms/ViewFn.java  |    5 +
 .../beam/sdk/transforms/WithTimestamps.java     |   26 +-
 .../sdk/transforms/display/DisplayData.java     |    6 +
 .../transforms/join/KeyedPCollectionTuple.java  |   12 +-
 .../reflect/ByteBuddyDoFnInvokerFactory.java    |   47 +-
 .../sdk/transforms/reflect/DoFnInvoker.java     |    4 +-
 .../sdk/transforms/reflect/DoFnSignature.java   |   10 +-
 .../sdk/transforms/reflect/DoFnSignatures.java  |   96 +-
 .../splittabledofn/HasDefaultTracker.java       |   30 +
 .../transforms/splittabledofn/OffsetRange.java  |    8 +-
 .../splittabledofn/OffsetRangeTracker.java      |   33 +-
 .../splittabledofn/RestrictionTracker.java      |    8 +
 .../transforms/windowing/AfterWatermark.java    |   16 +-
 .../sdk/transforms/windowing/GlobalWindow.java  |   10 +
 .../sdk/transforms/windowing/GlobalWindows.java |   11 +-
 .../transforms/windowing/InvalidWindows.java    |    2 +-
 .../beam/sdk/transforms/windowing/PaneInfo.java |    4 +-
 .../windowing/PartitioningWindowFn.java         |   17 +-
 .../sdk/transforms/windowing/Repeatedly.java    |    2 +-
 .../beam/sdk/transforms/windowing/Sessions.java |    2 +-
 .../transforms/windowing/SlidingWindows.java    |   27 +-
 .../beam/sdk/transforms/windowing/Triggers.java |  320 --
 .../beam/sdk/transforms/windowing/Window.java   |  515 ++-
 .../beam/sdk/transforms/windowing/WindowFn.java |    9 +-
 .../transforms/windowing/WindowMappingFn.java   |   67 +
 .../beam/sdk/util/AppEngineEnvironment.java     |   62 -
 ...AttemptAndTimeBoundedExponentialBackOff.java |  170 -
 .../util/AttemptBoundedExponentialBackOff.java  |   86 -
 .../org/apache/beam/sdk/util/CoderUtils.java    |   21 +-
 .../beam/sdk/util/CombineContextFactory.java    |    2 +-
 .../apache/beam/sdk/util/CredentialFactory.java |   29 -
 .../org/apache/beam/sdk/util/DefaultBucket.java |  105 -
 .../beam/sdk/util/DirectSideInputReader.java    |   74 -
 .../beam/sdk/util/FileIOChannelFactory.java     |   23 +-
 .../beam/sdk/util/GcpCredentialFactory.java     |   67 -
 .../apache/beam/sdk/util/GcpProjectUtil.java    |  106 -
 .../beam/sdk/util/GcsIOChannelFactory.java      |  110 -
 .../sdk/util/GcsIOChannelFactoryRegistrar.java  |   38 -
 .../apache/beam/sdk/util/GcsPathValidator.java  |   95 -
 .../java/org/apache/beam/sdk/util/GcsUtil.java  |  803 -----
 .../apache/beam/sdk/util/IOChannelFactory.java  |    3 +-
 .../apache/beam/sdk/util/IOChannelUtils.java    |   22 +-
 .../apache/beam/sdk/util/IdentityWindowFn.java  |    3 +-
 .../util/IntervalBoundedExponentialBackOff.java |   89 -
 .../apache/beam/sdk/util/MutationDetector.java  |    2 +-
 .../org/apache/beam/sdk/util/NameUtils.java     |    5 +
 .../beam/sdk/util/NoopCredentialFactory.java    |   68 -
 .../sdk/util/NullCredentialInitializer.java     |   62 -
 .../beam/sdk/util/NumberedShardedFile.java      |   30 +-
 .../apache/beam/sdk/util/PCollectionViews.java  |  268 +-
 .../java/org/apache/beam/sdk/util/PTuple.java   |  160 -
 .../org/apache/beam/sdk/util/PubsubClient.java  |  544 ---
 .../apache/beam/sdk/util/PubsubGrpcClient.java  |  424 ---
 .../apache/beam/sdk/util/PubsubJsonClient.java  |  317 --
 .../apache/beam/sdk/util/PubsubTestClient.java  |  436 ---
 .../apache/beam/sdk/util/ReifyTimestamps.java   |   76 +
 .../org/apache/beam/sdk/util/Reshuffle.java     |   21 +-
 .../apache/beam/sdk/util/TestCredential.java    |   59 -
 .../org/apache/beam/sdk/util/TimeDomain.java    |    2 +-
 .../org/apache/beam/sdk/util/Transport.java     |  179 -
 .../beam/sdk/util/WindowingStrategies.java      |  266 --
 .../util/state/AccumulatorCombiningState.java   |   53 -
 .../apache/beam/sdk/util/state/BagState.java    |    2 +-
 .../beam/sdk/util/state/CombiningState.java     |   27 +-
 .../beam/sdk/util/state/GroupingState.java      |   42 +
 .../apache/beam/sdk/util/state/MapState.java    |   52 +-
 .../beam/sdk/util/state/ReadableStates.java     |   45 +
 .../apache/beam/sdk/util/state/SetState.java    |   36 +-
 .../org/apache/beam/sdk/util/state/State.java   |    2 +-
 .../apache/beam/sdk/util/state/StateBinder.java |   12 +-
 .../apache/beam/sdk/util/state/StateSpec.java   |   15 +
 .../apache/beam/sdk/util/state/StateSpecs.java  |  344 +-
 .../beam/sdk/util/state/WatermarkHoldState.java |    2 +-
 .../java/org/apache/beam/sdk/values/PBegin.java |    6 +-
 .../apache/beam/sdk/values/PCollectionList.java |   27 +-
 .../beam/sdk/values/PCollectionTuple.java       |   13 +-
 .../apache/beam/sdk/values/PCollectionView.java |   19 +
 .../java/org/apache/beam/sdk/values/PDone.java  |    6 +-
 .../java/org/apache/beam/sdk/values/PInput.java |    4 +-
 .../org/apache/beam/sdk/values/POutput.java     |    4 +-
 .../beam/sdk/values/POutputValueBase.java       |   41 +-
 .../java/org/apache/beam/sdk/values/PValue.java |    4 +-
 .../org/apache/beam/sdk/values/PValueBase.java  |    6 +-
 .../apache/beam/sdk/values/TaggedPValue.java    |    5 +
 .../org/apache/beam/sdk/values/TupleTag.java    |   26 +-
 .../apache/beam/sdk/values/TupleTagList.java    |    2 +-
 .../apache/beam/sdk/values/TypeDescriptors.java |   25 +-
 .../org/apache/beam/sdk/values/TypedPValue.java |    6 +-
 .../beam/sdk/values/ValueInSingleWindow.java    |  134 +
 sdks/java/core/src/main/proto/README.md         |    3 -
 .../org/apache/beam/SdkCoreApiSurfaceTest.java  |    3 -
 .../sdk/AggregatorPipelineExtractorTest.java    |   56 +-
 .../java/org/apache/beam/sdk/PipelineTest.java  |  218 +-
 .../sdk/coders/protobuf/ProtobufUtilTest.java   |    1 -
 .../java/org/apache/beam/sdk/io/AvroIOTest.java |  154 +-
 .../org/apache/beam/sdk/io/AvroSourceTest.java  |   10 +-
 .../io/BoundedReadFromUnboundedSourceTest.java  |    8 +-
 .../apache/beam/sdk/io/CountingInputTest.java   |   14 +-
 .../apache/beam/sdk/io/CountingSourceTest.java  |   20 +-
 .../apache/beam/sdk/io/FileBasedSinkTest.java   |   94 +-
 .../apache/beam/sdk/io/FileBasedSourceTest.java |    8 +-
 .../org/apache/beam/sdk/io/FileSystemsTest.java |  168 +-
 .../apache/beam/sdk/io/LocalFileSystemTest.java |   12 +-
 .../beam/sdk/io/OffsetBasedSourceTest.java      |    8 +-
 .../org/apache/beam/sdk/io/PubsubIOTest.java    |  197 --
 .../beam/sdk/io/PubsubUnboundedSinkTest.java    |  190 -
 .../beam/sdk/io/PubsubUnboundedSourceTest.java  |  398 ---
 .../java/org/apache/beam/sdk/io/ReadTest.java   |   10 +-
 .../org/apache/beam/sdk/io/TFRecordIOTest.java  |  369 ++
 .../java/org/apache/beam/sdk/io/TextIOTest.java |  125 +-
 .../java/org/apache/beam/sdk/io/WriteTest.java  |   53 +-
 .../org/apache/beam/sdk/io/XmlSinkTest.java     |   89 +-
 .../org/apache/beam/sdk/io/XmlSourceTest.java   |  250 +-
 .../apache/beam/sdk/metrics/GaugeCellTest.java  |   48 +
 .../beam/sdk/metrics/MetricFilteringTest.java   |  145 +
 .../apache/beam/sdk/metrics/MetricMatchers.java |   12 +-
 .../apache/beam/sdk/metrics/MetricsTest.java    |   93 +-
 .../apache/beam/sdk/options/GcpOptionsTest.java |  171 -
 .../sdk/options/GoogleApiDebugOptionsTest.java  |  145 -
 .../sdk/options/PipelineOptionsFactoryTest.java |    4 +-
 .../beam/sdk/runners/PipelineRunnerTest.java    |   46 +-
 .../sdk/runners/TransformHierarchyTest.java     |   86 +-
 .../beam/sdk/runners/TransformTreeTest.java     |   88 +-
 .../runners/dataflow/TestCountingSource.java    |    2 +-
 .../beam/sdk/testing/BigqueryMatcherTest.java   |  176 -
 .../beam/sdk/testing/GatherAllPanesTest.java    |    1 +
 .../apache/beam/sdk/testing/PAssertTest.java    |   74 +-
 .../sdk/testing/PCollectionViewTesting.java     |   83 +-
 .../beam/sdk/testing/PaneExtractorsTest.java    |    1 +
 .../beam/sdk/testing/SourceTestUtilsTest.java   |    2 +-
 .../beam/sdk/testing/StaticWindowsTest.java     |   10 +-
 .../beam/sdk/testing/TestPipelineTest.java      |   47 +-
 .../apache/beam/sdk/testing/TestStreamTest.java |    6 +-
 .../testing/ValueInSingleWindowCoderTest.java   |    1 +
 .../sdk/transforms/ApproximateUniqueTest.java   |    4 +-
 .../beam/sdk/transforms/CombineFnsTest.java     |    8 +-
 .../apache/beam/sdk/transforms/CombineTest.java |   44 +-
 .../apache/beam/sdk/transforms/CountTest.java   |   10 +-
 .../apache/beam/sdk/transforms/CreateTest.java  |   70 +-
 .../beam/sdk/transforms/DistinctTest.java       |    8 +-
 .../beam/sdk/transforms/DoFnTesterTest.java     |    7 +-
 .../apache/beam/sdk/transforms/FilterTest.java  |   16 +-
 .../apache/beam/sdk/transforms/FlattenTest.java |   47 +-
 .../beam/sdk/transforms/GroupByKeyTest.java     |   14 +-
 .../sdk/transforms/GroupIntoBatchesTest.java    |  232 ++
 .../apache/beam/sdk/transforms/KeysTest.java    |    6 +-
 .../apache/beam/sdk/transforms/KvSwapTest.java  |    6 +-
 .../beam/sdk/transforms/MapElementsTest.java    |   29 +-
 .../beam/sdk/transforms/ParDoLifecycleTest.java |   10 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   | 1180 +++++--
 .../beam/sdk/transforms/PartitionTest.java      |    4 +-
 .../apache/beam/sdk/transforms/SampleTest.java  |   14 +-
 .../beam/sdk/transforms/SplittableDoFnTest.java |  157 +-
 .../beam/sdk/transforms/ToStringTest.java       |   13 +-
 .../org/apache/beam/sdk/transforms/TopTest.java |    3 +-
 .../apache/beam/sdk/transforms/ValuesTest.java  |    6 +-
 .../apache/beam/sdk/transforms/ViewTest.java    |  222 +-
 .../beam/sdk/transforms/WithTimestampsTest.java |    8 +-
 .../sdk/transforms/display/DisplayDataTest.java |   17 +
 .../sdk/transforms/join/CoGroupByKeyTest.java   |   10 +-
 .../transforms/reflect/DoFnInvokersTest.java    |  153 +-
 .../DoFnSignaturesProcessElementTest.java       |    2 +-
 .../DoFnSignaturesSplittableDoFnTest.java       |  117 +-
 .../splittabledofn/OffsetRangeTrackerTest.java  |   49 +-
 .../windowing/CalendarWindowsTest.java          |   33 +
 .../transforms/windowing/FixedWindowsTest.java  |   33 +
 .../windowing/SlidingWindowsTest.java           |   16 +-
 .../sdk/transforms/windowing/TriggersTest.java  |  100 -
 .../sdk/transforms/windowing/WindowTest.java    |   97 +-
 .../sdk/transforms/windowing/WindowingTest.java |   10 +-
 ...mptAndTimeBoundedExponentialBackOffTest.java |  213 --
 .../AttemptBoundedExponentialBackOffTest.java   |   85 -
 .../apache/beam/sdk/util/DefaultBucketTest.java |  112 -
 .../beam/sdk/util/GcpProjectUtilTest.java       |   76 -
 .../util/GcsIOChannelFactoryRegistrarTest.java  |   44 -
 .../beam/sdk/util/GcsIOChannelFactoryTest.java  |   43 -
 .../beam/sdk/util/GcsPathValidatorTest.java     |   87 -
 .../org/apache/beam/sdk/util/GcsUtilTest.java   |  798 -----
 .../sdk/util/IdentitySideInputWindowFn.java     |   10 +-
 .../IntervalBoundedExponentialBackOffTest.java  |  100 -
 .../org/apache/beam/sdk/util/NameUtilsTest.java |   12 +
 .../org/apache/beam/sdk/util/PTupleTest.java    |   40 -
 .../apache/beam/sdk/util/PubsubClientTest.java  |  189 -
 .../beam/sdk/util/PubsubGrpcClientTest.java     |  207 --
 .../beam/sdk/util/PubsubJsonClientTest.java     |  133 -
 .../beam/sdk/util/PubsubTestClientTest.java     |  114 -
 .../beam/sdk/util/ReifyTimestampsTest.java      |  109 +
 .../org/apache/beam/sdk/util/ReshuffleTest.java |   84 +-
 .../util/RetryHttpRequestInitializerTest.java   |  290 --
 .../beam/sdk/util/WindowingStrategiesTest.java  |   91 -
 .../beam/sdk/values/PCollectionListTest.java    |   70 +-
 .../beam/sdk/values/PCollectionTupleTest.java   |   17 +-
 .../org/apache/beam/sdk/values/PDoneTest.java   |    4 +-
 .../apache/beam/sdk/values/TypedPValueTest.java |   50 +-
 sdks/java/extensions/gcp-core/pom.xml           |  222 ++
 .../beam/sdk/options/BigQueryOptions.java       |   32 +
 .../options/CloudResourceManagerOptions.java    |   40 +
 .../org/apache/beam/sdk/options/GcpOptions.java |  227 ++
 .../options/GcpPipelineOptionsRegistrar.java    |   39 +
 .../org/apache/beam/sdk/options/GcsOptions.java |  154 +
 .../beam/sdk/options/GoogleApiDebugOptions.java |   87 +
 .../apache/beam/sdk/options/PubsubOptions.java  |   36 +
 .../apache/beam/sdk/options/package-info.java   |   22 +
 .../beam/sdk/testing/BigqueryMatcher.java       |  256 ++
 .../apache/beam/sdk/testing/package-info.java   |   21 +
 .../apache/beam/sdk/util/CredentialFactory.java |   29 +
 .../org/apache/beam/sdk/util/DefaultBucket.java |  105 +
 .../beam/sdk/util/GcpCredentialFactory.java     |   67 +
 .../apache/beam/sdk/util/GcpProjectUtil.java    |  106 +
 .../beam/sdk/util/GcsIOChannelFactory.java      |  111 +
 .../sdk/util/GcsIOChannelFactoryRegistrar.java  |   38 +
 .../apache/beam/sdk/util/GcsPathValidator.java  |   95 +
 .../java/org/apache/beam/sdk/util/GcsUtil.java  |  798 +++++
 .../beam/sdk/util/NoopCredentialFactory.java    |   68 +
 .../sdk/util/NullCredentialInitializer.java     |   62 +
 .../apache/beam/sdk/util/TestCredential.java    |   59 +
 .../org/apache/beam/sdk/util/Transport.java     |  178 +
 .../org/apache/beam/sdk/util/package-info.java  |   20 +
 .../org/apache/beam/GcpCoreApiSurfaceTest.java  |   62 +
 .../apache/beam/sdk/options/GcpOptionsTest.java |  171 +
 .../sdk/options/GoogleApiDebugOptionsTest.java  |  145 +
 .../beam/sdk/testing/BigqueryMatcherTest.java   |  176 +
 .../apache/beam/sdk/util/DefaultBucketTest.java |  112 +
 .../beam/sdk/util/GcpProjectUtilTest.java       |   76 +
 .../util/GcsIOChannelFactoryRegistrarTest.java  |   44 +
 .../beam/sdk/util/GcsIOChannelFactoryTest.java  |   43 +
 .../beam/sdk/util/GcsPathValidatorTest.java     |   87 +
 .../org/apache/beam/sdk/util/GcsUtilTest.java   |  798 +++++
 .../util/RetryHttpRequestInitializerTest.java   |  290 ++
 sdks/java/extensions/jackson/pom.xml            |   40 -
 sdks/java/extensions/pom.xml                    |    1 +
 sdks/java/extensions/sorter/pom.xml             |   49 -
 sdks/java/harness/pom.xml                       |   10 +-
 .../beam/fn/harness/fake/FakeStepContext.java   |    2 +-
 .../control/ProcessBundleHandlerTest.java       |   30 +-
 sdks/java/io/common/pom.xml                     |   42 +
 .../apache/beam/sdk/io/common/HashingFn.java    |  109 +
 .../sdk/io/common/IOTestPipelineOptions.java    |   79 +
 sdks/java/io/elasticsearch/pom.xml              |    7 +
 .../sdk/io/elasticsearch/ElasticsearchIO.java   |   37 +-
 .../sdk/io/elasticsearch/ElasticsearchIOIT.java |    9 +-
 .../io/elasticsearch/ElasticsearchIOTest.java   |   15 +-
 .../elasticsearch/ElasticsearchTestDataSet.java |   16 +-
 .../elasticsearch/ElasticsearchTestOptions.java |   46 -
 sdks/java/io/google-cloud-platform/pom.xml      |   93 +-
 .../beam/sdk/io/gcp/bigquery/BatchLoads.java    |  225 ++
 .../sdk/io/gcp/bigquery/BigQueryHelpers.java    |  352 ++
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    | 3306 ++++--------------
 .../io/gcp/bigquery/BigQueryQuerySource.java    |  205 ++
 .../io/gcp/bigquery/BigQueryServicesImpl.java   |    2 +-
 .../sdk/io/gcp/bigquery/BigQuerySourceBase.java |  205 ++
 .../gcp/bigquery/BigQueryTableRowIterator.java  |    3 +-
 .../io/gcp/bigquery/BigQueryTableSource.java    |  132 +
 .../beam/sdk/io/gcp/bigquery/CreateTables.java  |  127 +
 .../io/gcp/bigquery/GenerateShardedTable.java   |   47 +
 .../io/gcp/bigquery/PassThroughThenCleanup.java |   84 +
 .../beam/sdk/io/gcp/bigquery/PrepareWrite.java  |   81 +
 .../beam/sdk/io/gcp/bigquery/ShardedKey.java    |   67 +
 .../sdk/io/gcp/bigquery/ShardedKeyCoder.java    |   87 +
 .../sdk/io/gcp/bigquery/StreamingInserts.java   |   79 +
 .../sdk/io/gcp/bigquery/StreamingWriteFn.java   |  111 +
 .../io/gcp/bigquery/StreamingWriteTables.java   |   86 +
 .../sdk/io/gcp/bigquery/TableDestination.java   |   76 +
 .../io/gcp/bigquery/TableDestinationCoder.java  |   60 +
 .../beam/sdk/io/gcp/bigquery/TableRowInfo.java  |   34 +
 .../sdk/io/gcp/bigquery/TableRowInfoCoder.java  |   68 +
 .../sdk/io/gcp/bigquery/TableRowWriter.java     |   91 +
 .../sdk/io/gcp/bigquery/TagWithUniqueIds.java   |   62 +
 .../sdk/io/gcp/bigquery/TransformingSource.java |  136 +
 .../io/gcp/bigquery/WriteBundlesToFiles.java    |  157 +
 .../sdk/io/gcp/bigquery/WritePartition.java     |  174 +
 .../beam/sdk/io/gcp/bigquery/WriteRename.java   |  185 +
 .../beam/sdk/io/gcp/bigquery/WriteResult.java   |   46 +
 .../beam/sdk/io/gcp/bigquery/WriteTables.java   |  211 ++
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java    |  107 +-
 .../beam/sdk/io/gcp/datastore/DatastoreV1.java  |  398 ++-
 .../beam/sdk/io/gcp/pubsub/PubsubClient.java    |  544 +++
 .../sdk/io/gcp/pubsub/PubsubGrpcClient.java     |  424 +++
 .../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 1014 ++++++
 .../sdk/io/gcp/pubsub/PubsubJsonClient.java     |  319 ++
 .../sdk/io/gcp/pubsub/PubsubTestClient.java     |  436 +++
 .../sdk/io/gcp/pubsub/PubsubUnboundedSink.java  |  490 +++
 .../io/gcp/pubsub/PubsubUnboundedSource.java    | 1463 ++++++++
 .../beam/sdk/io/gcp/pubsub/package-info.java    |   24 +
 .../beam/sdk/io/gcp/storage/GcsResourceId.java  |    2 +-
 .../beam/sdk/io/gcp/GcpApiSurfaceTest.java      |    8 +-
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     | 1812 ++++------
 .../sdk/io/gcp/bigquery/BigQueryUtilTest.java   |   12 +-
 .../io/gcp/bigquery/FakeBigQueryServices.java   |  166 +
 .../sdk/io/gcp/bigquery/FakeDatasetService.java |  208 ++
 .../sdk/io/gcp/bigquery/FakeJobService.java     |  404 +++
 .../sdk/io/gcp/bigquery/TableContainer.java     |   61 +
 .../sdk/io/gcp/bigtable/BigtableIOTest.java     |   20 +-
 .../sdk/io/gcp/bigtable/BigtableWriteIT.java    |   15 +-
 .../sdk/io/gcp/datastore/DatastoreV1Test.java   |  252 +-
 .../sdk/io/gcp/datastore/SplitQueryFnIT.java    |    2 +-
 .../beam/sdk/io/gcp/datastore/V1ReadIT.java     |   66 +-
 .../sdk/io/gcp/datastore/V1TestOptions.java     |    2 +-
 .../sdk/io/gcp/pubsub/PubsubClientTest.java     |  189 +
 .../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java |  208 ++
 .../beam/sdk/io/gcp/pubsub/PubsubIOTest.java    |  189 +
 .../sdk/io/gcp/pubsub/PubsubJsonClientTest.java |  139 +
 .../sdk/io/gcp/pubsub/PubsubTestClientTest.java |  114 +
 .../io/gcp/pubsub/PubsubUnboundedSinkTest.java  |  188 +
 .../gcp/pubsub/PubsubUnboundedSourceTest.java   |  409 +++
 .../io/hadoop/SerializableConfiguration.java    |    3 +-
 sdks/java/io/hadoop-input-format/README.md      |  167 -
 sdks/java/io/hadoop-input-format/pom.xml        |  136 -
 .../hadoop/inputformat/HadoopInputFormatIO.java |  941 -----
 .../sdk/io/hadoop/inputformat/package-info.java |   23 -
 .../ConfigurableEmployeeInputFormat.java        |  131 -
 .../sdk/io/hadoop/inputformat/Employee.java     |   85 -
 .../hadoop/inputformat/EmployeeInputFormat.java |  172 -
 .../inputformat/HadoopInputFormatIOTest.java    |  844 -----
 .../ReuseObjectsEmployeeInputFormat.java        |  176 -
 .../hadoop/inputformat/TestEmployeeDataSet.java |   76 -
 sdks/java/io/hadoop/README.md                   |  167 +
 sdks/java/io/hadoop/input-format/pom.xml        |   98 +
 .../hadoop/inputformat/HadoopInputFormatIO.java |  844 +++++
 .../sdk/io/hadoop/inputformat/package-info.java |   23 +
 .../ConfigurableEmployeeInputFormat.java        |  131 +
 .../sdk/io/hadoop/inputformat/Employee.java     |   85 +
 .../hadoop/inputformat/EmployeeInputFormat.java |  172 +
 .../inputformat/HadoopInputFormatIOTest.java    |  851 +++++
 .../ReuseObjectsEmployeeInputFormat.java        |  176 +
 .../hadoop/inputformat/TestEmployeeDataSet.java |   76 +
 sdks/java/io/hadoop/jdk1.8-tests/pom.xml        |  244 ++
 .../inputformat/HIFIOWithElasticTest.java       |  277 ++
 .../HIFIOWithEmbeddedCassandraTest.java         |  215 ++
 .../custom/options/HIFTestOptions.java          |   64 +
 .../integration/tests/HIFIOCassandraIT.java     |  173 +
 .../integration/tests/HIFIOElasticIT.java       |  215 ++
 .../src/test/resources/cassandra.yaml           | 1074 ++++++
 sdks/java/io/hadoop/pom.xml                     |   64 +
 sdks/java/io/hbase/pom.xml                      |   92 +-
 .../org/apache/beam/sdk/io/hbase/HBaseIO.java   |    7 +-
 .../apache/beam/sdk/io/hbase/HBaseIOTest.java   |   13 +-
 .../src/test/resources/log4j-test.properties    |   27 +
 sdks/java/io/hdfs/README.md                     |    6 +-
 sdks/java/io/hdfs/pom.xml                       |   41 -
 .../apache/beam/sdk/io/hdfs/HDFSFileSink.java   |   29 +-
 .../apache/beam/sdk/io/hdfs/HDFSFileSource.java |   16 +-
 .../beam/sdk/io/hdfs/HDFSFileSinkTest.java      |    2 +-
 .../beam/sdk/io/hdfs/HDFSFileSourceTest.java    |   23 +-
 sdks/java/io/jdbc/pom.xml                       |   53 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java     |   75 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java   |   11 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java |   12 +-
 .../beam/sdk/io/jdbc/JdbcTestDataSet.java       |   12 +-
 .../beam/sdk/io/jdbc/PostgresTestOptions.java   |   60 -
 .../kubernetes/postgres-pod-no-vol.yml          |   32 -
 .../kubernetes/postgres-service-public.yml      |   28 -
 .../jdbc/src/test/resources/kubernetes/setup.sh |   20 -
 .../src/test/resources/kubernetes/teardown.sh   |   20 -
 .../java/org/apache/beam/sdk/io/jms/JmsIO.java  |    4 +-
 .../org/apache/beam/sdk/io/jms/JmsIOTest.java   |   10 +-
 sdks/java/io/kafka/pom.xml                      |   36 -
 .../apache/beam/sdk/io/kafka/ConsumerSpEL.java  |   43 +-
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java   |   70 +-
 .../apache/beam/sdk/io/kafka/KafkaRecord.java   |   15 +-
 .../beam/sdk/io/kafka/KafkaRecordCoder.java     |    5 +
 .../apache/beam/sdk/io/kafka/KafkaIOTest.java   |   54 +-
 .../apache/beam/sdk/io/kinesis/KinesisIO.java   |    2 +
 .../beam/sdk/io/kinesis/KinesisSource.java      |    2 +-
 .../beam/sdk/io/mongodb/MongoDbGridFSIO.java    |   14 +-
 .../apache/beam/sdk/io/mongodb/MongoDbIO.java   |    4 +-
 .../sdk/io/mongodb/MongoDBGridFSIOTest.java     |    7 +-
 .../beam/sdk/io/mongodb/MongoDbIOTest.java      |    5 -
 .../org/apache/beam/sdk/io/mqtt/MqttIO.java     |    4 +-
 .../org/apache/beam/sdk/io/mqtt/MqttIOTest.java |    6 -
 sdks/java/io/pom.xml                            |    3 +-
 .../beam/sdk/transforms/FilterJava8Test.java    |    8 +-
 .../transforms/FlatMapElementsJava8Test.java    |   10 +-
 .../sdk/transforms/MapElementsJava8Test.java    |   14 +-
 .../beam/sdk/transforms/WithKeysJava8Test.java  |    4 +-
 .../sdk/transforms/WithTimestampsJava8Test.java |    4 +-
 .../java/javadoc/dataflow-sdk-docs/package-list |   11 -
 sdks/java/javadoc/overview.html                 |    2 +-
 .../META-INF/maven/archetype-metadata.xml       |    2 +-
 .../META-INF/maven/archetype-metadata.xml       |    2 +-
 .../META-INF/maven/archetype-metadata.xml       |    2 +-
 sdks/python/.pylintrc                           |    6 +-
 sdks/python/MANIFEST.in                         |    3 -
 sdks/python/apache_beam/__init__.py             |    7 +-
 sdks/python/apache_beam/coders/coder_impl.py    |   50 +-
 sdks/python/apache_beam/coders/coders.py        |   10 +-
 sdks/python/apache_beam/coders/typecoders.py    |    2 +-
 sdks/python/apache_beam/error.py                |    4 +
 .../examples/complete/game/README.md            |   69 +
 .../examples/complete/game/__init__.py          |   16 +
 .../examples/complete/game/hourly_team_score.py |  294 ++
 .../complete/game/hourly_team_score_test.py     |   52 +
 .../examples/complete/game/user_score.py        |  217 ++
 .../examples/complete/game/user_score_test.py   |   49 +
 .../examples/complete/top_wikipedia_sessions.py |    8 -
 .../examples/cookbook/bigquery_tornadoes.py     |    2 +-
 .../cookbook/bigquery_tornadoes_it_test.py      |    3 +
 .../examples/cookbook/datastore_wordcount.py    |   15 +-
 .../examples/cookbook/group_with_coder.py       |    6 +-
 .../examples/cookbook/multiple_output_pardo.py  |   47 +-
 .../apache_beam/examples/snippets/snippets.py   |    5 +-
 .../examples/snippets/snippets_test.py          |   36 +-
 sdks/python/apache_beam/examples/wordcount.py   |   48 +-
 .../apache_beam/examples/wordcount_debugging.py |    4 -
 .../apache_beam/examples/wordcount_it_test.py   |   11 +-
 sdks/python/apache_beam/internal/gcp/auth.py    |   88 +-
 .../apache_beam/internal/gcp/auth_test.py       |   44 -
 .../apache_beam/internal/gcp/json_value.py      |    6 +
 sdks/python/apache_beam/internal/pickler.py     |   20 +-
 sdks/python/apache_beam/io/__init__.py          |    1 +
 sdks/python/apache_beam/io/avroio.py            |   12 +-
 sdks/python/apache_beam/io/avroio_test.py       |   14 +-
 sdks/python/apache_beam/io/concat_source.py     |   74 +-
 .../python/apache_beam/io/concat_source_test.py |   12 +-
 sdks/python/apache_beam/io/filebasedsource.py   |  127 +-
 .../apache_beam/io/filebasedsource_test.py      |   43 +-
 sdks/python/apache_beam/io/fileio.py            |  638 +---
 sdks/python/apache_beam/io/fileio_test.py       |  166 +-
 sdks/python/apache_beam/io/filesystem.py        |  530 +++
 sdks/python/apache_beam/io/filesystem_test.py   |  213 ++
 sdks/python/apache_beam/io/filesystems_util.py  |   35 +
 sdks/python/apache_beam/io/gcp/bigquery.py      |   94 +-
 sdks/python/apache_beam/io/gcp/bigquery_test.py |    3 +-
 .../io/gcp/datastore/v1/datastoreio.py          |   24 +-
 .../io/gcp/datastore/v1/datastoreio_test.py     |    4 +-
 .../apache_beam/io/gcp/datastore/v1/helper.py   |   18 +-
 .../io/gcp/datastore/v1/query_splitter.py       |    2 +-
 sdks/python/apache_beam/io/gcp/gcsfilesystem.py |  243 ++
 .../apache_beam/io/gcp/gcsfilesystem_test.py    |  322 ++
 sdks/python/apache_beam/io/gcp/gcsio.py         |   33 +-
 sdks/python/apache_beam/io/gcp/gcsio_test.py    |   48 +-
 .../io/gcp/tests/bigquery_matcher.py            |    3 +-
 sdks/python/apache_beam/io/iobase.py            |   20 +-
 sdks/python/apache_beam/io/localfilesystem.py   |  235 ++
 .../apache_beam/io/localfilesystem_test.py      |  189 +
 sdks/python/apache_beam/io/range_trackers.py    |   19 +-
 sdks/python/apache_beam/io/source_test_utils.py |   79 +-
 .../apache_beam/io/source_test_utils_test.py    |   20 +-
 sdks/python/apache_beam/io/textio.py            |   23 +-
 sdks/python/apache_beam/io/textio_test.py       |   20 +-
 sdks/python/apache_beam/io/tfrecordio.py        |   26 +-
 sdks/python/apache_beam/io/tfrecordio_test.py   |   38 +-
 sdks/python/apache_beam/metrics/cells.py        |   28 +-
 sdks/python/apache_beam/metrics/execution.py    |    3 +-
 sdks/python/apache_beam/metrics/metric.py       |    9 +-
 sdks/python/apache_beam/pipeline.py             |  138 +-
 sdks/python/apache_beam/pipeline_test.py        |   31 +-
 sdks/python/apache_beam/pvalue.py               |  317 +-
 sdks/python/apache_beam/pvalue_test.py          |   33 -
 .../runners/api/beam_runner_api_pb2.py          |  272 +-
 sdks/python/apache_beam/runners/common.pxd      |    2 +-
 sdks/python/apache_beam/runners/common.py       |   15 +-
 .../runners/dataflow/dataflow_metrics.py        |   86 +-
 .../runners/dataflow/dataflow_metrics_test.py   |   66 +-
 .../runners/dataflow/dataflow_runner.py         |  164 +-
 .../runners/dataflow/dataflow_runner_test.py    |   67 +-
 .../runners/dataflow/internal/apiclient.py      |   63 +-
 .../runners/dataflow/internal/apiclient_test.py |   12 +-
 .../runners/dataflow/internal/dependency.py     |   30 +-
 .../runners/dataflow/native_io/iobase_test.py   |  173 +
 .../runners/dataflow/test_dataflow_runner.py    |   26 +-
 .../runners/direct/bundle_factory.py            |   17 +-
 sdks/python/apache_beam/runners/direct/clock.py |    9 +-
 .../consumer_tracking_pipeline_visitor.py       |   11 +-
 .../consumer_tracking_pipeline_visitor_test.py  |   28 +-
 .../apache_beam/runners/direct/direct_runner.py |   22 +-
 .../runners/direct/evaluation_context.py        |   77 +-
 .../apache_beam/runners/direct/executor.py      |   28 +-
 .../runners/direct/transform_evaluator.py       |  116 +-
 .../runners/direct/transform_result.py          |   45 +-
 .../runners/direct/watermark_manager.py         |    4 +-
 sdks/python/apache_beam/runners/runner.py       |   16 +-
 .../apache_beam/tests/pipeline_verifiers.py     |   46 +-
 .../tests/pipeline_verifiers_test.py            |   47 +-
 sdks/python/apache_beam/transforms/combiners.py |   62 +-
 .../apache_beam/transforms/combiners_test.py    |    4 +-
 sdks/python/apache_beam/transforms/core.py      |  204 +-
 .../apache_beam/transforms/create_test.py       |  121 +
 sdks/python/apache_beam/transforms/display.py   |    4 +-
 .../apache_beam/transforms/display_test.py      |   36 +
 .../python/apache_beam/transforms/ptransform.py |   42 +-
 .../apache_beam/transforms/ptransform_test.py   |   41 +-
 .../python/apache_beam/transforms/sideinputs.py |  143 +-
 .../apache_beam/transforms/sideinputs_test.py   |   99 +-
 sdks/python/apache_beam/transforms/trigger.py   |   40 +-
 .../apache_beam/transforms/trigger_test.py      |    6 +-
 sdks/python/apache_beam/transforms/util.py      |   20 +-
 sdks/python/apache_beam/transforms/util_test.py |   50 +
 sdks/python/apache_beam/transforms/window.py    |    9 +-
 .../apache_beam/transforms/window_test.py       |    8 +-
 sdks/python/apache_beam/typehints/decorators.py |   26 +-
 .../apache_beam/typehints/trivial_inference.py  |   30 +-
 .../typehints/trivial_inference_test.py         |    3 +-
 sdks/python/apache_beam/typehints/typecheck.py  |   11 +-
 sdks/python/apache_beam/typehints/typehints.py  |   66 +-
 .../apache_beam/typehints/typehints_test.py     |    7 +-
 .../apache_beam/utils/annotations_test.py       |    2 +-
 sdks/python/apache_beam/utils/counters.py       |    5 +-
 sdks/python/apache_beam/utils/path.py           |    3 +-
 .../apache_beam/utils/pipeline_options.py       |  106 +-
 .../apache_beam/utils/pipeline_options_test.py  |   52 +-
 sdks/python/apache_beam/utils/proto_utils.py    |   17 +-
 sdks/python/apache_beam/utils/retry.py          |   20 +-
 sdks/python/apache_beam/utils/test_stream.py    |  163 +
 .../apache_beam/utils/test_stream_test.py       |   82 +
 sdks/python/apache_beam/utils/timestamp.py      |    6 +-
 sdks/python/apache_beam/utils/urns.py           |    2 +
 sdks/python/apache_beam/utils/value_provider.py |  103 +
 .../apache_beam/utils/value_provider_test.py    |  145 +
 sdks/python/apache_beam/utils/windowed_value.py |   17 +-
 sdks/python/apache_beam/version.py              |   36 -
 sdks/python/findSupportedPython.groovy          |   80 +
 sdks/python/generate_pydoc.sh                   |    2 +-
 sdks/python/pom.xml                             |   31 +-
 sdks/python/run_postcommit.sh                   |   35 +-
 sdks/python/run_pylint.sh                       |   27 +-
 sdks/python/setup.py                            |   19 +-
 sdks/python/tox.ini                             |   18 +-
 1164 files changed, 69329 insertions(+), 54774 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
----------------------------------------------------------------------
diff --cc .test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
index 0000000,1348a19..e1cbafe
mode 000000,100644..100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
@@@ -1,0 -1,49 +1,49 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ import common_job_properties
+ 
+ // This job runs the suite of ValidatesRunner tests against the Gearpump
+ // runner.
+ mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
+   description('Runs the ValidatesRunner suite on the Gearpump runner.')
+ 
+   previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump')
+ 
+   // Set common parameters.
+   common_job_properties.setTopLevelMainJobProperties(
+       delegate,
+       'gearpump-runner')
+ 
+   // Set maven parameters.
+   common_job_properties.setMavenConfig(delegate)
+ 
+   // Sets that this is a PostCommit job.
+   // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
+   // In post-commit this job triggers only on SCM changes.
+   common_job_properties.setPostCommit(delegate, '0 5 31 2 *')
+ 
+   // Allows triggering this build against pull requests.
+   common_job_properties.enablePhraseTriggeringFromPullRequest(
+     delegate,
+     'Apache Gearpump Runner ValidatesRunner Tests',
+     'Run Gearpump ValidatesRunner')
+ 
+   // Maven goals for this job.
 -  goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'')
++  goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner"]\'')
+ }

http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/.travis.yml
----------------------------------------------------------------------
diff --cc .travis.yml
index 87ade2f,0b5d700..a1b28f9
--- a/.travis.yml
+++ b/.travis.yml
@@@ -78,11 -77,10 +77,11 @@@ install
    - if [ "$TEST_PYTHON" ] && ! pip list | grep tox; then travis_retry pip install tox --user; fi
    # Removing this here protects from inadvertent caching
    - rm -rf "$HOME/.m2/repository/org/apache/beam"
 +  - rm -rf "$HOME/.m2/repository/org/apache/gearpump"
  
  script:
-   - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -e $TOX_ENV -c sdks/python/tox.ini; fi
-   - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh; fi
+   - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -c sdks/python/tox.ini; fi
+   - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .test-infra/travis/test_wordcount.sh; fi
  
  cache:
    directories:

http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/examples/java/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
----------------------------------------------------------------------
diff --cc runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
index 0000000,0000000..e78568d
new file mode 100644
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoMultiOutputTranslator.java
@@@ -1,0 -1,0 +1,104 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements.  See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership.  The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License.  You may obtain a copy of the License at
++ *
++ *     http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.beam.runners.gearpump.translators;
++
++import java.util.ArrayList;
++import java.util.Collection;
++import java.util.List;
++import java.util.Map;
++
++import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
++import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
++import org.apache.beam.sdk.transforms.DoFn;
++import org.apache.beam.sdk.transforms.ParDo;
++import org.apache.beam.sdk.util.WindowedValue;
++import org.apache.beam.sdk.values.PCollection;
++import org.apache.beam.sdk.values.PCollectionView;
++import org.apache.beam.sdk.values.TaggedPValue;
++import org.apache.beam.sdk.values.TupleTag;
++
++import org.apache.gearpump.streaming.dsl.api.functions.FilterFunction;
++import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
++
++/**
++ * {@link ParDo.MultiOutput} is translated to Gearpump flatMap function
++ * with {@link DoFn} wrapped in {@link DoFnFunction}. The outputs are
++ * further filtered with Gearpump filter function by output tag
++ */
++@SuppressWarnings({"rawtypes", "unchecked"})
++public class ParDoMultiOutputTranslator<InputT, OutputT> implements
++    TransformTranslator<ParDo.MultiOutput<InputT, OutputT>> {
++
++  private static final long serialVersionUID = -6023461558200028849L;
++
++  @Override
++  public void translate(ParDo.MultiOutput<InputT, OutputT> transform, TranslationContext context) {
++    PCollection<InputT> inputT = (PCollection<InputT>) context.getInput();
++    JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(inputT);
++    Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
++    Map<String, PCollectionView<?>> tagsToSideInputs =
++        TranslatorUtils.getTagsToSideInputs(sideInputs);
++
++    List<TaggedPValue> outputs = context.getOutputs();
++    final TupleTag<OutputT> mainOutput = transform.getMainOutputTag();
++    List<TupleTag<?>> sideOutputs = new ArrayList<>(outputs.size() - 1);
++    for (TaggedPValue output: outputs) {
++      TupleTag<?> tag = output.getTag();
++      if (tag != null && !tag.getId().equals(mainOutput.getId())) {
++        sideOutputs.add(tag);
++      }
++    }
++
++    JavaStream<TranslatorUtils.RawUnionValue> unionStream = TranslatorUtils.withSideInputStream(
++        context, inputStream, tagsToSideInputs);
++
++    JavaStream<TranslatorUtils.RawUnionValue> outputStream =
++        TranslatorUtils.toList(unionStream).flatMap(
++            new DoFnFunction<>(
++                context.getPipelineOptions(),
++                transform.getFn(),
++                inputT.getWindowingStrategy(),
++                sideInputs,
++                tagsToSideInputs,
++                mainOutput,
++                sideOutputs), transform.getName());
++    for (TaggedPValue output: outputs) {
++      JavaStream<WindowedValue<OutputT>> taggedStream = outputStream
++          .filter(new FilterByOutputTag(output.getTag().getId()),
++              "filter_by_output_tag")
++          .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
++      context.setOutputStream(output.getValue(), taggedStream);
++    }
++  }
++
++  private static class FilterByOutputTag extends FilterFunction<TranslatorUtils.RawUnionValue> {
++
++    private static final long serialVersionUID = 7276155265895637526L;
++    private final String tag;
++
++    FilterByOutputTag(String tag) {
++      this.tag = tag;
++    }
++
++    @Override
++    public boolean filter(TranslatorUtils.RawUnionValue value) {
++      return value.getUnionTag().equals(tag);
++    }
++  }
++}

http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
----------------------------------------------------------------------
diff --cc runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
index 0000000,0000000..86879b7
new file mode 100644
--- /dev/null
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/ParDoSingleOutputTranslator.java
@@@ -1,0 -1,0 +1,75 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements.  See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership.  The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License.  You may obtain a copy of the License at
++ *
++ *     http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.beam.runners.gearpump.translators;
++
++import java.util.Collection;
++import java.util.List;
++import java.util.Map;
++
++import org.apache.beam.runners.gearpump.translators.functions.DoFnFunction;
++import org.apache.beam.runners.gearpump.translators.utils.TranslatorUtils;
++import org.apache.beam.sdk.transforms.DoFn;
++import org.apache.beam.sdk.transforms.ParDo;
++import org.apache.beam.sdk.util.WindowedValue;
++import org.apache.beam.sdk.util.WindowingStrategy;
++import org.apache.beam.sdk.values.PCollection;
++
++import org.apache.beam.sdk.values.PCollectionView;
++import org.apache.beam.sdk.values.TupleTag;
++import org.apache.beam.sdk.values.TupleTagList;
++import org.apache.gearpump.streaming.dsl.javaapi.JavaStream;
++
++/**
++ * {@link ParDo.SingleOutput} is translated to Gearpump flatMap function
++ * with {@link DoFn} wrapped in {@link DoFnFunction}.
++ */
++public class ParDoSingleOutputTranslator<InputT, OutputT> implements
++    TransformTranslator<ParDo.SingleOutput<InputT, OutputT>> {
++
++  private static final long serialVersionUID = -3413205558160983784L;
++  private final TupleTag<OutputT> mainOutput = new TupleTag<>();
++  private final List<TupleTag<?>> sideOutputs = TupleTagList.empty().getAll();
++
++  @Override
++  public void translate(ParDo.SingleOutput<InputT, OutputT> transform, TranslationContext context) {
++    DoFn<InputT, OutputT> doFn = transform.getFn();
++    PCollection<OutputT> output = (PCollection<OutputT>) context.getOutput();
++    WindowingStrategy<?, ?> windowingStrategy = output.getWindowingStrategy();
++
++    Collection<PCollectionView<?>> sideInputs = transform.getSideInputs();
++    Map<String, PCollectionView<?>> tagsToSideInputs =
++        TranslatorUtils.getTagsToSideInputs(sideInputs);
++    JavaStream<WindowedValue<InputT>> inputStream = context.getInputStream(
++        context.getInput());
++    JavaStream<TranslatorUtils.RawUnionValue> unionStream =
++        TranslatorUtils.withSideInputStream(context,
++        inputStream, tagsToSideInputs);
++
++    DoFnFunction<InputT, OutputT> doFnFunction = new DoFnFunction<>(context.getPipelineOptions(),
++        doFn, windowingStrategy, sideInputs, tagsToSideInputs,
++        mainOutput, sideOutputs);
++
++    JavaStream<WindowedValue<OutputT>> outputStream =
++        TranslatorUtils.toList(unionStream)
++            .flatMap(doFnFunction, transform.getName())
++            .map(new TranslatorUtils.FromRawUnionValue<OutputT>(), "from_RawUnionValue");
++
++    context.setOutputStream(context.getOutput(), outputStream);
++  }
++}

http://git-wip-us.apache.org/repos/asf/beam/blob/46c41fce/runners/pom.xml
----------------------------------------------------------------------
diff --cc runners/pom.xml
index dd791ff,8f3cabd..a114382
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@@ -54,55 -54,6 +54,15 @@@
          </plugins>
        </build>
      </profile>
 +    <profile>
 +      <id>java8</id>
 +      <activation>
 +        <jdk>[1.8,)</jdk>
 +      </activation>
 +      <modules>
 +        <module>gearpump</module>
 +      </modules>
 +    </profile>
- 
-     <!-- A profile that adds an integration test phase if and only if
-          the runnableOnServicePipelineOptions maven property has been set.
-          It should be set to a valid PipelineOptions JSON string. -->
-     <profile>
-       <id>runnable-on-service-tests</id>
-       <activation>
-         <property><name>runnableOnServicePipelineOptions</name></property>
-       </activation>
-       <build>
-         <pluginManagement>
-           <plugins>
-             <plugin>
-               <groupId>org.apache.maven.plugins</groupId>
-               <artifactId>maven-surefire-plugin</artifactId>
-               <executions>
-                 <execution>
-                   <id>runnable-on-service-tests</id>
-                   <phase>integration-test</phase>
-                   <goals>
-                     <goal>test</goal>
-                   </goals>
-                   <configuration>
-                     <groups>org.apache.beam.sdk.testing.RunnableOnService</groups>
-                     <parallel>all</parallel>
-                     <threadCount>4</threadCount>
-                     <dependenciesToScan>
-                       <dependency>org.apache.beam:beam-sdks-java-core</dependency>
-                     </dependenciesToScan>
-                     <systemPropertyVariables>
-                       <beamTestPipelineOptions>${runnableOnServicePipelineOptions}</beamTestPipelineOptions>
-                     </systemPropertyVariables>
-                   </configuration>
-                 </execution>
-               </executions>
-             </plugin>
-           </plugins>
-         </pluginManagement>
-       </build>
-     </profile>
    </profiles>
  
    <build>


Mime
View raw message