beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lc...@apache.org
Subject [beam] 01/01: Merge master into go-sdk
Date Thu, 01 Feb 2018 01:23:55 GMT
This is an automated email from the ASF dual-hosted git repository.

lcwik pushed a commit to branch go-sdk
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 010272b08c977da959dd8d93e5645691bdda02e1
Merge: 8cbd440 5386431
Author: Lukasz Cwik <lcwik@google.com>
AuthorDate: Wed Jan 31 17:23:47 2018 -0800

    Merge master into go-sdk

 .gitignore                                         |  67 +-
 .test-infra/jenkins/common_job_properties.groovy   |  13 +-
 ...job_beam_PerformanceTests_FileBasedIO_IT.groovy | 134 ++-
 ...job_beam_PostCommit_Java_JDKVersionsTest.groovy |   4 +-
 ...eam_PostCommit_Java_MavenInstall_Windows.groovy |   2 +-
 .../job_beam_PostRelease_NightlySnapshot.groovy    |  52 ++
 .../hdfs-single-datanode-cluster-for-local-dev.yml |  38 +-
 .../hdfs-single-datanode-cluster.yml               |  83 ++
 .../hadoop/SmallITCluster/pkb-config.yml           |  40 +
 .../kubernetes/hadoop/SmallITCluster/setup-all.sh  |  42 +
 .../kubernetes/hadoop/SmallITCluster/setup.sh      |  12 +-
 .../hadoop/SmallITCluster/teardown-all.sh          |  25 +-
 .../kubernetes/hadoop/SmallITCluster/teardown.sh   |  14 +-
 .../kubernetes/hadoop/config-files/core-site.xml   |  15 +-
 .../kubernetes/hadoop/config-files/hdfs-envs       |  15 +-
 .../kubernetes/hadoop/config-files/hdfs-site.xml   |  21 +-
 CONTRIBUTING.md                                    |  24 +
 NOTICE                                             |   2 +-
 assembly.xml                                       | 111 +++
 build.gradle                                       |  67 +-
 build_rules.gradle                                 |  77 +-
 examples/java/README.md                            |  10 +-
 examples/java/build.gradle                         |  32 +-
 examples/java/pom.xml                              |  11 +-
 .../org/apache/beam/examples/MinimalWordCount.java |  88 +-
 .../apache/beam/examples/WindowedWordCount.java    |   4 +-
 .../java/org/apache/beam/examples/WordCount.java   |   3 +-
 .../apache/beam/examples/common/ExampleUtils.java  |  63 +-
 .../beam/examples/complete/AutoComplete.java       |  78 +-
 .../org/apache/beam/examples/complete/TfIdf.java   |  50 +-
 .../examples/complete/TopWikipediaSessions.java    |  17 +-
 .../beam/examples/complete/TrafficMaxLaneFlow.java |  14 +-
 .../beam/examples/complete/TrafficRoutes.java      |  17 +-
 .../beam/examples/complete/game/GameStats.java     | 172 ++--
 .../examples/complete/game/HourlyTeamScore.java    |  80 +-
 .../beam/examples/complete/game/LeaderBoard.java   |  50 +-
 .../examples/complete/game/StatefulTeamScore.java  | 227 +++++
 .../beam/examples/complete/game/UserScore.java     |  18 +-
 .../examples/complete/game/injector/Injector.java  |  91 +-
 .../complete/game/injector/InjectorUtils.java      |   0
 .../game/injector/RetryHttpInitializerWrapper.java |  42 +-
 .../complete/game/utils/GameConstants.java         |  20 +-
 .../complete/game/utils/WriteToBigQuery.java       |   0
 .../examples/complete/game/utils/WriteToText.java  |   2 +-
 .../game/utils/WriteWindowedToBigQuery.java        |   0
 .../beam/examples/cookbook/BigQueryTornadoes.java  |   3 +-
 .../examples/cookbook/CombinePerKeyExamples.java   |   4 +-
 .../beam/examples/cookbook/DistinctExample.java    |   4 +-
 .../beam/examples/cookbook/FilterExamples.java     |   3 +-
 .../beam/examples/cookbook/JoinExamples.java       |  12 +-
 .../beam/examples/cookbook/MaxPerKeyExamples.java  |   3 +-
 .../beam/examples/cookbook/TriggerExample.java     |   4 +-
 .../apache/beam/examples/snippets}/Snippets.java   |  13 +-
 .../beam/examples/MinimalWordCountTest.java}       |  57 +-
 .../apache/beam/examples/WindowedWordCountIT.java  |   8 +-
 .../org/apache/beam/examples/WordCountTest.java    |   3 +-
 .../beam/examples/complete/AutoCompleteTest.java   |  17 +-
 .../apache/beam/examples/complete/TfIdfTest.java   |   4 +-
 .../beam/examples/complete/game/GameStatsTest.java |   0
 .../complete/game/HourlyTeamScoreTest.java         |   0
 .../examples/complete/game/LeaderBoardTest.java    |  16 +-
 .../complete/game/StatefulTeamScoreTest.java       | 208 +++++
 .../beam/examples/complete/game/UserScoreTest.java |   0
 .../examples/cookbook/DistinctExampleTest.java     |   6 +-
 .../beam/examples/cookbook/TriggerExampleTest.java |   6 +-
 .../beam/examples/snippets}/SnippetsTest.java      |   5 +-
 examples/java8/build.gradle                        |  50 --
 examples/java8/pom.xml                             | 292 -------
 .../beam/examples/MinimalWordCountJava8.java       |  72 --
 examples/pom.xml                                   |  25 +-
 gradle.properties                                  |   1 +
 model/fn-execution/build.gradle                    |   2 +-
 model/fn-execution/pom.xml                         |   2 +-
 .../fn-execution/src/main/proto/beam_fn_api.proto  |   3 +
 model/job-management/build.gradle                  |   2 +-
 model/job-management/pom.xml                       |  20 +-
 model/pipeline/pom.xml                             |   2 +-
 model/pom.xml                                      |   2 +-
 pom.xml                                            | 424 ++++-----
 release/TestScripts.groovy                         | 140 +++
 release/quickstart-java-direct.groovy              |  64 ++
 runners/apex/build.gradle                          |  18 +-
 runners/apex/pom.xml                               |   3 +-
 .../org/apache/beam/runners/apex/ApexRunner.java   |  22 +-
 .../beam/runners/apex/ApexRunnerRegistrar.java     |   5 +-
 .../apache/beam/runners/apex/ApexYarnLauncher.java |  19 +-
 .../runners/apex/translation/ParDoTranslator.java  |   2 +-
 .../operators/ApexGroupByKeyOperator.java          |  43 +-
 .../translation/operators/ApexParDoOperator.java   |  10 +-
 .../operators/ApexProcessFnOperator.java           |   6 +-
 .../operators/ApexReadUnboundedInputOperator.java  |   2 +-
 .../apex/translation/utils/ApexStateInternals.java |   9 +-
 .../apache/beam/runners/apex/ApexRunnerTest.java   |   2 +-
 .../beam/runners/apex/ApexYarnLauncherTest.java    |   8 +-
 .../beam/runners/apex/examples/WordCountTest.java  |  14 +-
 .../FlattenPCollectionTranslatorTest.java          |   9 +-
 .../apex/translation/GroupByKeyTranslatorTest.java |   4 +-
 .../apex/translation/ParDoTranslatorTest.java      |  52 +-
 .../apex/translation/SideInputTranslationTest.java |  10 +-
 .../operators/ApexTimerInternalsTest.java          |   7 +-
 runners/core-construction-java/build.gradle        |   6 +-
 runners/core-construction-java/pom.xml             |   2 +-
 .../core/construction/ArtifactServiceStager.java   |   3 +-
 .../core/construction/CoderTranslation.java        |  88 +-
 .../runners/core/construction/CoderTranslator.java |  12 +-
 .../construction/CoderTranslatorRegistrar.java     |  26 +-
 .../core/construction/CoderTranslators.java        |  19 +-
 .../core/construction/CombineTranslation.java      |  87 +-
 .../CreatePCollectionViewTranslation.java          |   5 +-
 .../construction/DeduplicatedFlattenFactory.java   |   6 +-
 .../construction/EmptyFlattenAsCreateFactory.java  |   2 +-
 .../runners/core/construction/Environments.java    | 107 +++
 .../core/construction/ForwardingPTransform.java    |   8 +
 .../core/construction/ModelCoderRegistrar.java     |  79 ++
 .../construction/PCollectionViewTranslation.java   | 110 +++
 .../core/construction/PTransformMatchers.java      |  45 +-
 .../core/construction/PTransformTranslation.java   |   3 +-
 .../core/construction/ParDoTranslation.java        | 231 ++---
 .../core/construction/PipelineTranslation.java     |  13 +-
 .../runners/core/construction/PrimitiveCreate.java |   2 +-
 .../runners/core/construction/ReadTranslation.java |  35 +-
 .../core/construction/RehydratedComponents.java    |  17 +-
 .../runners/core/construction/SdkComponents.java   |  28 +-
 .../runners/core/construction/SplittableParDo.java |  22 +-
 .../UnboundedReadFromBoundedSource.java            |  20 +-
 .../runners/core/construction/UnconsumedReads.java |   6 +-
 .../core/construction/WindowIntoTranslation.java   |   4 +-
 .../construction/WindowingStrategyTranslation.java |  28 +-
 .../core/construction/WriteFilesTranslation.java   |  13 +-
 .../construction/ArtifactServiceStagerTest.java    |   2 +-
 .../core/construction/CoderTranslationTest.java    |  23 +-
 .../core/construction/CombineTranslationTest.java  |   2 +-
 .../CreatePCollectionViewTranslationTest.java      |  12 +-
 .../DeduplicatedFlattenFactoryTest.java            |   8 +-
 .../EmptyFlattenAsCreateFactoryTest.java           |  31 +-
 .../core/construction/EnvironmentsTest.java        | 224 +++++
 .../construction/ForwardingPTransformTest.java     |  38 +-
 .../construction/PCollectionTranslationTest.java   |  22 +-
 .../PCollectionViewTranslationTest.java            |  74 ++
 .../core/construction/PTransformMatchersTest.java  | 146 ++--
 .../construction/PTransformReplacementsTest.java   |  12 +-
 .../construction/PTransformTranslationTest.java    |   9 +-
 .../core/construction/ParDoTranslationTest.java    |  46 +-
 .../core/construction/PipelineTranslationTest.java |   9 +-
 .../core/construction/ReadTranslationTest.java     |  10 +-
 .../construction/RehydratedComponentsTest.java     |  14 +
 .../core/construction/ReplacementOutputsTest.java  |   9 +-
 .../core/construction/SdkComponentsTest.java       |  36 +-
 .../SingleInputOutputOverrideFactoryTest.java      |   2 +-
 .../construction/TestStreamTranslationTest.java    |   8 +-
 .../core/construction/TransformInputsTest.java     |  29 +-
 .../UnboundedReadFromBoundedSourceTest.java        |  18 +-
 .../core/construction/UnconsumedReadsTest.java     |   9 +-
 .../UnsupportedOverrideFactoryTest.java            |   4 +-
 .../WindowingStrategyTranslationTest.java          |   2 +-
 .../construction/metrics/MetricFilteringTest.java  |   2 +-
 runners/core-java/build.gradle                     |  10 +-
 runners/core-java/pom.xml                          |   2 +-
 .../runners/core/GroupByKeyViaGroupByKeyOnly.java  |  24 +-
 .../beam/runners/core/InMemoryStateInternals.java  |   4 +-
 .../apache/beam/runners/core/KeyedWorkItems.java   |   2 +-
 .../runners/core/LateDataDroppingDoFnRunner.java   |  51 +-
 .../apache/beam/runners/core/LateDataUtils.java    |  63 +-
 .../beam/runners/core/MergingActiveWindowSet.java  |  13 +-
 .../beam/runners/core/NullSideInputReader.java     |   2 +-
 ...TimeBoundedSplittableProcessElementInvoker.java |   9 +-
 .../beam/runners/core/ReduceFnContextFactory.java  |  15 +-
 .../apache/beam/runners/core/ReduceFnRunner.java   |  54 +-
 .../apache/beam/runners/core/SideInputHandler.java |   3 +-
 .../core/SplittableParDoViaKeyedWorkItems.java     |   2 +-
 .../apache/beam/runners/core/SystemReduceFn.java   |  21 +-
 .../apache/beam/runners/core/WatermarkHold.java    |   2 +-
 .../beam/runners/core/metrics/MetricUpdates.java   |   7 +-
 .../runners/core/metrics/MetricsContainerImpl.java |  25 +-
 .../core/metrics/MetricsContainerStepMap.java      | 159 ++--
 .../core/triggers/AfterAllStateMachine.java        |   2 +-
 .../AfterDelayFromFirstElementStateMachine.java    |  12 +-
 .../core/triggers/AfterEachStateMachine.java       |   2 +-
 .../core/triggers/AfterFirstStateMachine.java      |   2 +-
 ...fterSynchronizedProcessingTimeStateMachine.java |   3 +-
 .../core/triggers/AfterWatermarkStateMachine.java  |   4 +-
 .../TriggerStateMachineContextFactory.java         |  11 +-
 .../beam/runners/core/KeyedWorkItemCoderTest.java  |   5 +-
 .../core/LateDataDroppingDoFnRunnerTest.java       |  22 +-
 .../beam/runners/core/LateDataUtilsTest.java       |   3 +-
 .../beam/runners/core/ReduceFnRunnerTest.java      | 119 +--
 .../apache/beam/runners/core/ReduceFnTester.java   |  83 +-
 .../beam/runners/core/SideInputHandlerTest.java    |  49 +-
 .../beam/runners/core/SimpleDoFnRunnerTest.java    |  24 +-
 .../SimplePushbackSideInputDoFnRunnerTest.java     |  21 +-
 .../runners/core/SplittableParDoProcessFnTest.java |  22 +-
 .../beam/runners/core/StatefulDoFnRunnerTest.java  |  15 +-
 .../apache/beam/runners/core/WindowMatchers.java   |  20 +-
 .../core/metrics/MetricsContainerStepMapTest.java  |   2 +-
 .../beam/runners/core/metrics/MetricsMapTest.java  |   8 +-
 .../core/triggers/AfterFirstStateMachineTest.java  |   2 +-
 .../triggers/AfterWatermarkStateMachineTest.java   |   4 +-
 .../core/triggers/FinishedTriggersSetTest.java     |   9 +-
 .../core/triggers/RepeatedlyStateMachineTest.java  |   7 +-
 .../core/triggers/StubTriggerStateMachine.java     |   2 +-
 .../core/triggers/TriggerStateMachineTest.java     |  10 +-
 .../core/triggers/TriggerStateMachineTester.java   |  67 +-
 runners/direct-java/build.gradle                   |  37 +-
 runners/direct-java/pom.xml                        |   3 +-
 .../runners/direct/DirectExecutionContext.java     |   7 +-
 ...DirectGBKIntoKeyedWorkItemsOverrideFactory.java |   2 +-
 .../beam/runners/direct/DirectGroupByKey.java      |   4 +-
 .../apache/beam/runners/direct/DirectMetrics.java  |  45 +-
 .../beam/runners/direct/DirectRegistrar.java       |   6 +-
 .../beam/runners/direct/EvaluationContext.java     |   8 +-
 .../direct/ExecutorServiceParallelExecutor.java    |  13 +-
 .../direct/GroupAlsoByWindowEvaluatorFactory.java  |  52 +-
 .../direct/GroupByKeyOnlyEvaluatorFactory.java     |   7 +-
 .../direct/ImmutabilityEnforcementFactory.java     |   2 +-
 .../runners/direct/KeyedPValueTrackingVisitor.java |   2 +-
 .../beam/runners/direct/MultiStepCombine.java      |   2 +-
 .../apache/beam/runners/direct/ParDoEvaluator.java |  44 +-
 .../beam/runners/direct/ParDoEvaluatorFactory.java |   2 +-
 .../runners/direct/ParDoMultiOverrideFactory.java  |   8 +-
 .../beam/runners/direct/QuiescenceDriver.java      |  12 +-
 .../beam/runners/direct/ReadEvaluatorFactory.java  |   4 +-
 .../beam/runners/direct/SideInputContainer.java    |  27 +-
 .../SplittableProcessElementsEvaluatorFactory.java |  78 +-
 .../direct/StatefulParDoEvaluatorFactory.java      |  31 +-
 .../runners/direct/TestStreamEvaluatorFactory.java |   2 +-
 .../direct/UnboundedReadEvaluatorFactory.java      |  17 +-
 .../beam/runners/direct/ViewOverrideFactory.java   |  13 +-
 .../beam/runners/direct/WatermarkManager.java      |  78 +-
 .../runners/direct/WindowEvaluatorFactory.java     |   4 +-
 .../runners/direct/WriteWithShardingFactory.java   |   8 +-
 .../direct/BoundedReadEvaluatorFactoryTest.java    |  19 +-
 .../runners/direct/CloningBundleFactoryTest.java   |   9 +-
 .../beam/runners/direct/CommittedResultTest.java   |  20 +-
 .../runners/direct/DirectGraphVisitorTest.java     |  31 +-
 .../DirectGroupByKeyOverrideFactoryTest.java       |   3 +-
 .../beam/runners/direct/DirectMetricsTest.java     |  88 +-
 .../beam/runners/direct/DirectRunnerTest.java      | 107 ++-
 .../direct/DirectTransformExecutorTest.java        |  31 +-
 ...cycleManagerRemovingTransformEvaluatorTest.java |   6 +-
 .../runners/direct/DoFnLifecycleManagerTest.java   |   2 +-
 .../beam/runners/direct/EvaluationContextTest.java |  62 +-
 .../direct/FlattenEvaluatorFactoryTest.java        |  12 +-
 .../direct/GroupByKeyEvaluatorFactoryTest.java     |   2 +-
 .../direct/GroupByKeyOnlyEvaluatorFactoryTest.java |   2 +-
 .../ImmutabilityCheckingBundleFactoryTest.java     |   2 +-
 .../direct/ImmutabilityEnforcementFactoryTest.java |   8 +-
 .../direct/ImmutableListBundleFactoryTest.java     |  10 +-
 .../direct/KeyedPValueTrackingVisitorTest.java     |  23 +-
 .../beam/runners/direct/MultiStepCombineTest.java  |   8 +-
 .../beam/runners/direct/ParDoEvaluatorTest.java    |  14 +-
 .../runners/direct/SideInputContainerTest.java     | 117 +--
 .../direct/StatefulParDoEvaluatorFactoryTest.java  |  32 +-
 .../runners/direct/StepTransformResultTest.java    |   3 +-
 .../beam/runners/direct/StructuralKeyTest.java     |   4 +-
 .../direct/TestStreamEvaluatorFactoryTest.java     |   7 +-
 .../direct/UnboundedReadEvaluatorFactoryTest.java  |  30 +-
 .../runners/direct/ViewEvaluatorFactoryTest.java   |  15 +-
 .../runners/direct/ViewOverrideFactoryTest.java    |  25 +-
 .../beam/runners/direct/WatermarkManagerTest.java  | 392 ++++++---
 .../runners/direct/WindowEvaluatorFactoryTest.java |  21 +-
 .../direct/WriteWithShardingFactoryTest.java       |   5 +-
 runners/flink/build.gradle                         |  82 +-
 runners/flink/pom.xml                              |   8 +-
 .../runners/flink/CreateStreamingFlinkView.java    |  29 +-
 .../flink/FlinkBatchPipelineTranslator.java        |   5 +-
 .../flink/FlinkBatchTransformTranslators.java      | 192 +++--
 .../flink/FlinkPipelineExecutionEnvironment.java   |   9 +
 .../beam/runners/flink/FlinkRunnerRegistrar.java   |   7 +-
 .../beam/runners/flink/FlinkRunnerResult.java      |   7 +-
 .../flink/FlinkStreamingPipelineTranslator.java    |   6 +-
 .../flink/FlinkStreamingTransformTranslators.java  | 328 ++++---
 .../runners/flink/FlinkTransformOverrides.java     |   9 +-
 .../flink/metrics/FlinkMetricContainer.java        |   8 +-
 .../FlinkMergingNonShuffleReduceFunction.java      |   2 +-
 .../functions/FlinkPartialReduceFunction.java      |   3 +-
 .../translation/functions/FlinkReduceFunction.java |   2 +-
 .../functions/FlinkSideInputReader.java            |   7 +-
 .../functions/SideInputInitializer.java            |  31 +-
 .../functions/SortingFlinkCombineRunner.java       |  12 +-
 .../wrappers/streaming/DoFnOperator.java           |  28 +-
 .../wrappers/streaming/SplittableDoFnOperator.java |  35 +-
 .../wrappers/streaming/WindowDoFnOperator.java     |  36 +-
 .../state/FlinkBroadcastStateInternals.java        |   2 +-
 .../state/FlinkKeyGroupStateInternals.java         |   6 +-
 .../streaming/state/FlinkSplitStateInternals.java  |   2 +-
 .../streaming/state/FlinkStateInternals.java       |  39 +-
 .../beam/runners/flink/PipelineOptionsTest.java    |  52 +-
 .../flink/streaming/DedupingOperatorTest.java      |  46 +-
 .../runners/flink/streaming/DoFnOperatorTest.java  | 296 +++----
 .../flink/streaming/GroupByNullKeyTest.java        |  76 +-
 .../streaming/TopWikipediaSessionsITCase.java      | 122 ++-
 .../streaming/UnboundedSourceWrapperTest.java      |  78 +-
 runners/gcp/gcemd/pom.xml                          |   2 +-
 runners/gcp/gcsproxy/pom.xml                       |   2 +-
 runners/gcp/pom.xml                                |   2 +-
 runners/gearpump/build.gradle                      |  10 +-
 runners/gearpump/pom.xml                           |  26 +-
 .../runners/gearpump/GearpumpPipelineResult.java   |   7 +-
 .../runners/gearpump/GearpumpRunnerRegistrar.java  |   7 +-
 .../translators/CreateStreamingGearpumpView.java   |  10 +-
 .../translators/FlattenPCollectionsTranslator.java |   5 +-
 .../gearpump/translators/GroupByKeyTranslator.java |  23 +-
 runners/google-cloud-dataflow-java/build.gradle    |  28 +-
 runners/google-cloud-dataflow-java/pom.xml         |  31 +-
 .../dataflow/BatchStatefulParDoOverrides.java      |  22 +-
 .../beam/runners/dataflow/BatchViewOverrides.java  | 184 ++--
 .../beam/runners/dataflow/DataflowMetrics.java     |   2 +-
 .../beam/runners/dataflow/DataflowPipelineJob.java | 100 ++-
 .../dataflow/DataflowPipelineRegistrar.java        |   6 +-
 .../dataflow/DataflowPipelineTranslator.java       | 160 ++--
 .../beam/runners/dataflow/DataflowRunner.java      |  84 +-
 .../dataflow/PrimitiveParDoSingleFactory.java      | 174 +++-
 .../beam/runners/dataflow/ReadTranslator.java      |   2 +-
 .../runners/dataflow/ReshuffleOverrideFactory.java |  30 +-
 .../runners/dataflow/StreamingViewOverrides.java   |   2 +-
 .../beam/runners/dataflow/TransformTranslator.java |  22 +-
 .../beam/runners/dataflow/internal/IsmFormat.java  |  11 +-
 .../dataflow/util/CloudObjectTranslators.java      |  22 +-
 ...DefaultCoderCloudObjectTranslatorRegistrar.java |   2 +-
 .../beam/runners/dataflow/util/MonitoringUtil.java |  15 +-
 .../beam/runners/dataflow/util/PackageUtil.java    |  61 +-
 .../apache/beam/runners/dataflow/util/Structs.java |  10 +-
 .../dataflow/BatchStatefulParDoOverridesTest.java  |  13 +-
 .../runners/dataflow/BatchViewOverridesTest.java   |  12 +-
 .../beam/runners/dataflow/DataflowMetricsTest.java |   2 +-
 .../runners/dataflow/DataflowPipelineJobTest.java  |  50 +-
 .../dataflow/DataflowPipelineTranslatorTest.java   | 245 +-----
 .../beam/runners/dataflow/DataflowRunnerTest.java  | 272 ++++--
 .../dataflow/PrimitiveParDoSingleFactoryTest.java  |   2 +-
 .../runners/dataflow/TestDataflowRunnerTest.java   |  50 +-
 .../transforms/DataflowGroupByKeyTest.java         |  14 +-
 .../dataflow/transforms/DataflowViewTest.java      |  49 +-
 .../runners/dataflow/util/CloudObjectsTest.java    |   8 +-
 .../runners/dataflow/util/MonitoringUtilTest.java  |  12 +-
 .../runners/dataflow/util/PackageUtilTest.java     |  13 +-
 .../beam/runners/dataflow/util/StructsTest.java    |  11 +-
 runners/java-fn-execution/build.gradle             |  15 +-
 runners/java-fn-execution/pom.xml                  |  29 +-
 .../fnexecution/control/FnApiControlClient.java    |  10 +-
 .../control/FnApiControlClientPoolService.java     |   9 +-
 .../fnexecution/control/SdkHarnessClient.java      | 238 +++--
 .../runners/fnexecution/data/FnDataService.java    |  10 +-
 .../runners/fnexecution/data/GrpcDataService.java  | 166 ++++
 .../graph/LengthPrefixUnknownCoders.java           | 183 ++++
 .../runners/fnexecution/graph}/package-info.java   |   7 +-
 .../provisioning/StaticGrpcProvisionService.java   |  55 ++
 .../fnexecution/provisioning/package-info.java     |  21 +-
 .../runners/fnexecution/ServerFactoryTest.java     |  33 +-
 .../control/FnApiControlClientTest.java            |  21 +
 .../fnexecution/control/SdkHarnessClientTest.java  | 253 +++++-
 .../fnexecution/data/GrpcDataServiceTest.java      | 185 ++++
 .../graph/LengthPrefixUnknownCodersTest.java       | 147 ++++
 .../logging/GrpcLoggingServiceTest.java            | 112 ++-
 .../StaticGrpcProvisionServiceTest.java            |  97 +++
 runners/local-artifact-service-java/build.gradle   |   4 +-
 runners/local-artifact-service-java/pom.xml        |   2 +-
 .../LocalFileSystemArtifactStagerServiceTest.java  |   2 +-
 runners/local-java/build.gradle                    |   2 +-
 runners/local-java/pom.xml                         |   2 +-
 .../beam/runners/local/StructuralKeyTest.java      |   5 +-
 runners/pom.xml                                    |  12 +-
 runners/reference/java/build.gradle                |   4 +-
 runners/reference/java/pom.xml                     |   2 +-
 runners/reference/job-server/build.gradle          |  10 +-
 runners/reference/job-server/pom.xml               |   2 +-
 .../reference/job/ReferenceRunnerJobService.java   |   7 +-
 .../job/ReferenceRunnerJobServiceTest.java         |  12 +-
 runners/reference/pom.xml                          |   2 +-
 runners/spark/build.gradle                         |  20 +-
 runners/spark/pom.xml                              | 127 ++-
 .../runners/spark/SparkNativePipelineVisitor.java  |  15 +-
 .../org/apache/beam/runners/spark/SparkRunner.java |  22 +-
 .../beam/runners/spark/SparkRunnerRegistrar.java   |   5 +-
 .../apache/beam/runners/spark/TestSparkRunner.java |   2 +-
 .../spark/aggregators/AggregatorsAccumulator.java  |   6 +-
 .../spark/aggregators/NamedAggregators.java        |  19 +-
 .../beam/runners/spark/coders/CoderHelpers.java    |  66 +-
 .../beam/runners/spark/examples/WordCount.java     |   3 +-
 .../apache/beam/runners/spark/io/ConsoleIO.java    |  23 +-
 .../apache/beam/runners/spark/io/CreateStream.java |   3 +-
 .../beam/runners/spark/io/SourceDStream.java       |   4 +-
 .../apache/beam/runners/spark/io/SourceRDD.java    |   5 +-
 .../runners/spark/io/SparkUnboundedSource.java     |  18 +-
 .../runners/spark/metrics/MetricsAccumulator.java  |   6 +-
 .../runners/spark/metrics/WithMetricsSupport.java  |  95 +-
 .../SparkGroupAlsoByWindowViaWindowSet.java        |  99 +--
 .../spark/stateful/SparkTimerInternals.java        |   3 +-
 .../runners/spark/translation/BoundedDataset.java  |  19 +-
 .../spark/translation/EvaluationContext.java       |   4 +-
 .../spark/translation/GroupCombineFunctions.java   | 145 ++--
 .../spark/translation/MultiDoFnFunction.java       |  20 +-
 .../spark/translation/SparkAbstractCombineFn.java  |  10 +-
 .../spark/translation/SparkGlobalCombineFn.java    |  26 +-
 .../spark/translation/SparkKeyedCombineFn.java     |  28 +-
 .../spark/translation/SparkProcessContext.java     |   8 +-
 .../spark/translation/TransformTranslator.java     | 274 +++---
 .../spark/translation/TranslationUtils.java        | 178 ++--
 .../spark/translation/WindowingHelpers.java        |  29 +-
 .../streaming/StreamingTransformTranslator.java    | 174 ++--
 .../translation/streaming/UnboundedDataset.java    |  11 +-
 .../streaming/WatermarkSyncedDStream.java          |   3 +-
 .../runners/spark/util/GlobalWatermarkHolder.java  |   2 +-
 .../runners/spark/util/SparkSideInputReader.java   |  38 +-
 .../org/apache/beam/runners/spark/CacheTest.java   |   4 +-
 .../runners/spark/SparkRunnerDebuggerTest.java     |  14 +-
 .../spark/TestSparkPipelineOptionsRegistrar.java   |   2 +-
 .../aggregators/metrics/sink/InMemoryMetrics.java  |  18 +-
 .../metrics/sink/SparkMetricsSinkTest.java         |  65 +-
 .../beam/runners/spark/io/NumShardsTest.java       |  12 +-
 .../translation/streaming/CreateStreamTest.java    |  97 +--
 .../ResumeFromCheckpointStreamingTest.java         |  66 +-
 .../streaming/TrackStreamingSourcesTest.java       |   3 +-
 sdks/go/BUILD.md                                   |   2 +-
 .../beam/model/fnexecution_v1/beam_fn_api.pb.go    | 527 +++++++----
 sdks/go/pom.xml                                    |   2 +-
 sdks/java/build-tools/pom.xml                      |   2 +-
 .../src/main/resources/beam/findbugs-filter.xml    |  21 +-
 .../main/resources/docker/file/openjdk7/Dockerfile |  49 --
 sdks/java/container/build.gradle                   |   6 +-
 sdks/java/container/pom.xml                        |   2 +-
 sdks/java/core/build.gradle                        |  23 +-
 sdks/java/core/pom.xml                             |   3 +-
 .../main/java/org/apache/beam/sdk/Pipeline.java    |  73 +-
 .../java/org/apache/beam/sdk/coders/AvroCoder.java |   8 +-
 .../java/org/apache/beam/sdk/coders/ByteCoder.java |   2 +-
 .../sdk/coders/CannotProvideCoderException.java    |  10 +-
 .../java/org/apache/beam/sdk/coders/Coder.java     |   8 +-
 .../org/apache/beam/sdk/coders/CoderRegistry.java  |  54 +-
 .../org/apache/beam/sdk/coders/DefaultCoder.java   |   2 +-
 .../org/apache/beam/sdk/coders/DelegateCoder.java  |   2 +-
 .../beam/sdk/coders/StringDelegateCoder.java       |  29 +-
 .../org/apache/beam/sdk/coders/VarIntCoder.java    |   2 +-
 .../org/apache/beam/sdk/coders/VarLongCoder.java   |   4 +-
 .../main/java/org/apache/beam/sdk/io/AvroIO.java   |  22 +-
 .../main/java/org/apache/beam/sdk/io/AvroSink.java |   4 +-
 .../java/org/apache/beam/sdk/io/AvroSource.java    |  12 +-
 .../sdk/io/BoundedReadFromUnboundedSource.java     |  21 +-
 .../org/apache/beam/sdk/io/CompressedSource.java   |   3 +-
 .../beam/sdk/io/DynamicAvroDestinations.java       |   2 +-
 .../java/org/apache/beam/sdk/io/FileBasedSink.java |   4 +-
 .../main/java/org/apache/beam/sdk/io/FileIO.java   | 122 +--
 .../java/org/apache/beam/sdk/io/FileSystems.java   | 103 +--
 .../org/apache/beam/sdk/io/LocalFileSystem.java    |  23 +-
 .../beam/sdk/io/LocalFileSystemRegistrar.java      |   2 +-
 .../org/apache/beam/sdk/io/LocalResources.java     |   8 +-
 .../src/main/java/org/apache/beam/sdk/io/Read.java |   4 +-
 .../beam/sdk/io/ReadAllViaFileBasedSource.java     |   4 +-
 .../java/org/apache/beam/sdk/io/TFRecordIO.java    |   2 +-
 .../main/java/org/apache/beam/sdk/io/TextIO.java   |  35 +-
 .../java/org/apache/beam/sdk/io/WriteFiles.java    |  20 +-
 .../org/apache/beam/sdk/io/WriteFilesResult.java   |   4 +-
 .../beam/sdk/metrics/MetricsEnvironment.java       |   3 +-
 .../beam/sdk/options/PipelineOptionsFactory.java   | 137 ++-
 .../beam/sdk/options/ProxyInvocationHandler.java   |  23 +-
 .../org/apache/beam/sdk/options/ValueProvider.java |   2 +-
 .../apache/beam/sdk/runners/AppliedPTransform.java |   3 +-
 .../beam/sdk/runners/TransformHierarchy.java       |   8 +-
 .../java/org/apache/beam/sdk/state/StateSpecs.java |  12 +-
 .../org/apache/beam/sdk/testing/Annotations.java   |  34 +-
 .../apache/beam/sdk/testing/CoderProperties.java   |  11 +-
 .../apache/beam/sdk/testing/GatherAllPanes.java    |   6 +-
 .../java/org/apache/beam/sdk/testing/PAssert.java  | 103 +--
 .../beam/sdk/testing/SerializableMatchers.java     | 400 ++-------
 .../apache/beam/sdk/testing/SourceTestUtils.java   |  44 +-
 .../org/apache/beam/sdk/testing/TestPipeline.java  |   9 +-
 .../org/apache/beam/sdk/testing/TestStream.java    |  14 +-
 .../apache/beam/sdk/testing/WindowFnTestUtils.java |  21 +-
 .../beam/sdk/transforms/ApproximateQuantiles.java  |  31 +-
 .../beam/sdk/transforms/ApproximateUnique.java     |   3 +-
 .../org/apache/beam/sdk/transforms/Combine.java    |  43 +-
 .../org/apache/beam/sdk/transforms/Contextful.java |   9 +-
 .../java/org/apache/beam/sdk/transforms/Count.java |  25 +-
 .../org/apache/beam/sdk/transforms/Create.java     |  24 +-
 .../org/apache/beam/sdk/transforms/Distinct.java   |  41 +-
 .../org/apache/beam/sdk/transforms/DoFnTester.java |  51 +-
 .../org/apache/beam/sdk/transforms/Filter.java     |  41 +-
 .../org/apache/beam/sdk/transforms/Latest.java     |  51 +-
 .../java/org/apache/beam/sdk/transforms/Max.java   |  22 +-
 .../java/org/apache/beam/sdk/transforms/Mean.java  |   8 +-
 .../java/org/apache/beam/sdk/transforms/Min.java   |  22 +-
 .../java/org/apache/beam/sdk/transforms/ParDo.java |  33 +-
 .../java/org/apache/beam/sdk/transforms/Reify.java |   4 +-
 .../beam/sdk/transforms/ReifyTimestamps.java       |   4 +-
 .../apache/beam/sdk/transforms/Requirements.java   |   2 +-
 .../org/apache/beam/sdk/transforms/Reshuffle.java  |  18 +-
 .../org/apache/beam/sdk/transforms/Sample.java     |  14 +-
 .../beam/sdk/transforms/SerializableFunctions.java |   7 +-
 .../java/org/apache/beam/sdk/transforms/Sum.java   |   6 +-
 .../java/org/apache/beam/sdk/transforms/Top.java   |   2 +-
 .../java/org/apache/beam/sdk/transforms/View.java  |  24 +-
 .../java/org/apache/beam/sdk/transforms/Watch.java |  30 +-
 .../org/apache/beam/sdk/transforms/WithKeys.java   |   9 +-
 .../apache/beam/sdk/transforms/WithTimestamps.java |   2 +-
 .../beam/sdk/transforms/display/DisplayData.java   |  13 +-
 .../beam/sdk/transforms/join/CoGbkResult.java      |  11 +-
 .../beam/sdk/transforms/join/CoGroupByKey.java     |  15 +-
 .../sdk/transforms/join/KeyedPCollectionTuple.java |   2 +-
 .../reflect/ByteBuddyDoFnInvokerFactory.java       |  47 +-
 .../reflect/ByteBuddyOnTimerInvokerFactory.java    |  68 +-
 .../beam/sdk/transforms/reflect/DoFnSignature.java |  20 +-
 .../sdk/transforms/reflect/DoFnSignatures.java     |  95 +-
 .../beam/sdk/transforms/windowing/AfterAll.java    |   2 +-
 .../beam/sdk/transforms/windowing/AfterEach.java   |   2 +-
 .../beam/sdk/transforms/windowing/AfterFirst.java  |   2 +-
 .../transforms/windowing/AfterProcessingTime.java  |   2 +-
 .../sdk/transforms/windowing/AfterWatermark.java   |   7 +-
 .../beam/sdk/transforms/windowing/Trigger.java     |   8 +-
 .../beam/sdk/transforms/windowing/Window.java      |   2 +-
 .../java/org/apache/beam/sdk/util/ApiSurface.java  |  84 +-
 .../apache/beam/sdk/util/BucketingFunction.java    |   6 +-
 .../java/org/apache/beam/sdk/util/ClassPath.java   |   8 +-
 .../java/org/apache/beam/sdk/util/CoderUtils.java  |  12 +-
 .../apache/beam/sdk/util/DoFnAndMainOutput.java}   |  22 +-
 .../java/org/apache/beam/sdk/util/DoFnInfo.java    |  22 +-
 .../org/apache/beam/sdk/util/MovingFunction.java   |   4 +-
 .../java/org/apache/beam/sdk/util/Sleeper.java     |   7 +-
 .../java/org/apache/beam/sdk/util/StreamUtils.java |   2 +-
 .../org/apache/beam/sdk/util/WindowedValue.java    |   2 +-
 .../java/org/apache/beam/sdk/util/ZipFiles.java    |  11 +-
 .../beam/sdk/util/common/ReflectHelpers.java       |  62 +-
 .../org/apache/beam/sdk/values/PCollection.java    |   4 +-
 .../apache/beam/sdk/values/PCollectionList.java    |   2 +-
 .../apache/beam/sdk/values/PCollectionTuple.java   |   4 +-
 .../apache/beam/sdk/values/PCollectionViews.java   |   9 +-
 .../org/apache/beam/sdk/values/TupleTagList.java   |   2 +-
 .../org/apache/beam/sdk/values/TypeDescriptor.java |   2 +-
 .../apache/beam/sdk/values/TypeDescriptors.java    |  14 +-
 .../apache/beam/sdk/values/ValueWithRecordId.java  |   5 +-
 .../apache/beam/sdk/values/WindowingStrategy.java  |  98 ++-
 .../java/org/apache/beam/sdk/PipelineTest.java     | 129 +--
 .../org/apache/beam/sdk/coders/AvroCoderTest.java  |   4 +-
 .../beam/sdk/coders/BigDecimalCoderTest.java       |   4 +-
 .../apache/beam/sdk/coders/CoderProvidersTest.java |  13 +-
 .../apache/beam/sdk/coders/CoderRegistryTest.java  |  12 +
 .../java/org/apache/beam/sdk/coders/CoderTest.java |   2 +-
 .../beam/sdk/coders/CollectionCoderTest.java       |  15 +-
 .../apache/beam/sdk/coders/CommonCoderTest.java    |   2 +-
 .../apache/beam/sdk/coders/DefaultCoderTest.java   |   9 +-
 .../apache/beam/sdk/coders/DelegateCoderTest.java  |  36 +-
 .../apache/beam/sdk/coders/InstantCoderTest.java   |   2 +-
 .../apache/beam/sdk/coders/IterableCoderTest.java  |  14 +-
 .../org/apache/beam/sdk/coders/KvCoderTest.java    |   2 +-
 .../org/apache/beam/sdk/coders/ListCoderTest.java  |  20 +-
 .../org/apache/beam/sdk/coders/MapCoderTest.java   |   9 +-
 .../apache/beam/sdk/coders/NullableCoderTest.java  |  16 +-
 .../org/apache/beam/sdk/coders/SetCoderTest.java   |   9 +-
 .../java/org/apache/beam/sdk/io/AvroIOTest.java    |  87 +-
 .../org/apache/beam/sdk/io/AvroSourceTest.java     |  18 +-
 .../org/apache/beam/sdk/io/CountingSourceTest.java |  33 +-
 .../org/apache/beam/sdk/io/FileBasedSinkTest.java  |  31 +-
 .../apache/beam/sdk/io/FileBasedSourceTest.java    |  30 +-
 .../java/org/apache/beam/sdk/io/FileIOTest.java    |  79 +-
 .../org/apache/beam/sdk/io/FileSystemsTest.java    |  14 +-
 .../apache/beam/sdk/io/GenerateSequenceTest.java   |  19 +-
 .../apache/beam/sdk/io/LocalFileSystemTest.java    |  37 +-
 .../java/org/apache/beam/sdk/io/SimpleSink.java    |   7 +-
 .../org/apache/beam/sdk/io/TextIOReadTest.java     |  20 +-
 .../org/apache/beam/sdk/io/TextIOWriteTest.java    |  71 +-
 .../org/apache/beam/sdk/io/WriteFilesTest.java     |  26 +-
 .../sdk/options/PipelineOptionsFactoryTest.java    |   8 +-
 .../sdk/options/PipelineOptionsReflectorTest.java  |   5 +-
 .../beam/sdk/options/PipelineOptionsTest.java      |   2 +-
 .../sdk/options/ProxyInvocationHandlerTest.java    |  40 +-
 .../apache/beam/sdk/options/ValueProviderTest.java |  24 +-
 .../beam/sdk/options/ValueProvidersTest.java       |   4 +-
 .../beam/sdk/runners/PipelineRunnerTest.java       |  26 +-
 .../beam/sdk/runners/TransformHierarchyTest.java   |  42 +-
 .../apache/beam/sdk/runners/TransformTreeTest.java |   4 +-
 .../sdk/runners/dataflow/TestCountingSource.java   |  15 +-
 .../apache/beam/sdk/testing/ExpectedLogsTest.java  |  22 +-
 .../beam/sdk/testing/GatherAllPanesTest.java       |  65 +-
 .../org/apache/beam/sdk/testing/PAssertTest.java   |  82 +-
 .../beam/sdk/testing/SerializableMatchersTest.java |  12 +-
 .../apache/beam/sdk/testing/StaticWindowsTest.java |  29 +-
 .../apache/beam/sdk/testing/TestPipelineTest.java  |  10 +-
 .../apache/beam/sdk/testing/TestStreamTest.java    |  80 +-
 .../beam/sdk/testing/WindowSupplierTest.java       |   9 +-
 .../sdk/transforms/ApproximateQuantilesTest.java   |  45 +-
 .../beam/sdk/transforms/ApproximateUniqueTest.java |  20 +-
 .../apache/beam/sdk/transforms/CombineFnsTest.java |  92 +-
 .../apache/beam/sdk/transforms/CombineTest.java    | 155 ++--
 .../org/apache/beam/sdk/transforms/CountTest.java  |  12 +-
 .../org/apache/beam/sdk/transforms/CreateTest.java |  32 +-
 .../apache/beam/sdk/transforms/DistinctTest.java   |  36 +-
 .../apache/beam/sdk/transforms/DoFnTesterTest.java |   2 +-
 .../beam/sdk/transforms/FlatMapElementsTest.java   |  48 +-
 .../apache/beam/sdk/transforms/FlattenTest.java    |  69 +-
 .../apache/beam/sdk/transforms/GroupByKeyTest.java | 217 +++--
 .../beam/sdk/transforms/GroupIntoBatchesTest.java  |  85 +-
 .../org/apache/beam/sdk/transforms/KeysTest.java   |   4 +-
 .../org/apache/beam/sdk/transforms/KvSwapTest.java |   6 +-
 .../org/apache/beam/sdk/transforms/LatestTest.java |  34 +-
 .../beam/sdk/transforms/MapElementsTest.java       |  68 +-
 .../org/apache/beam/sdk/transforms/MeanTest.java   |   5 +-
 .../beam/sdk/transforms/ParDoLifecycleTest.java    |  25 +-
 .../org/apache/beam/sdk/transforms/ParDoTest.java  | 515 +++++++----
 .../apache/beam/sdk/transforms/PartitionTest.java  |   2 +-
 .../org/apache/beam/sdk/transforms/ReifyTest.java  |  21 +-
 .../beam/sdk/transforms/ReifyTimestampsTest.java   |  15 +-
 .../apache/beam/sdk/transforms/ReshuffleTest.java  | 130 ++-
 .../org/apache/beam/sdk/transforms/SampleTest.java |  56 +-
 .../beam/sdk/transforms/SimpleStatsFnsTest.java    |  14 +-
 .../beam/sdk/transforms/SplittableDoFnTest.java    |  61 +-
 .../org/apache/beam/sdk/transforms/TopTest.java    |  52 +-
 .../org/apache/beam/sdk/transforms/ValuesTest.java |   4 +-
 .../org/apache/beam/sdk/transforms/ViewTest.java   | 959 ++++++++++++---------
 .../org/apache/beam/sdk/transforms/WatchTest.java  | 173 ++--
 .../apache/beam/sdk/transforms/WithKeysTest.java   |   6 +-
 .../beam/sdk/transforms/WithTimestampsTest.java    |  47 +-
 .../transforms/display/DisplayDataMatchers.java    |  22 +-
 .../display/DisplayDataMatchersTest.java           |  68 +-
 .../sdk/transforms/join/CoGbkResultCoderTest.java  |  20 +-
 .../beam/sdk/transforms/join/CoGbkResultTest.java  |   8 +-
 .../beam/sdk/transforms/join/CoGroupByKeyTest.java | 112 ++-
 .../beam/sdk/transforms/join/UnionCoderTest.java   |  13 +-
 .../sdk/transforms/reflect/DoFnInvokersTest.java   |  13 +-
 .../reflect/DoFnSignaturesSplittableDoFnTest.java  |  10 +-
 .../sdk/transforms/reflect/DoFnSignaturesTest.java |   2 +-
 .../transforms/windowing/CalendarWindowsTest.java  |   2 +-
 .../sdk/transforms/windowing/FixedWindowsTest.java |   2 +-
 .../sdk/transforms/windowing/SessionsTest.java     |  10 +-
 .../beam/sdk/transforms/windowing/StubTrigger.java |   2 +-
 .../beam/sdk/transforms/windowing/TriggerTest.java |  10 +-
 .../beam/sdk/transforms/windowing/WindowTest.java  | 102 ++-
 .../sdk/transforms/windowing/WindowingTest.java    |  12 +-
 .../org/apache/beam/sdk/util/ApiSurfaceTest.java   |  11 +-
 .../BufferedElementCountingOutputStreamTest.java   |   2 +-
 .../org/apache/beam/sdk/util/NameUtilsTest.java    |   7 +-
 .../beam/sdk/util/NumberedShardedFileTest.java     |  10 +-
 .../apache/beam/sdk/util/WindowedValueTest.java    |   3 +-
 .../org/apache/beam/sdk/util/ZipFilesTest.java     |   5 +-
 .../beam/sdk/util/common/ReflectHelpersTest.java   |  16 +-
 .../beam/sdk/values/PCollectionListTest.java       |   3 +-
 .../beam/sdk/values/PCollectionTupleTest.java      |   2 +-
 .../org/apache/beam/sdk/values/TupleTagTest.java   |   4 +-
 .../apache/beam/sdk/values/TypedPValueTest.java    |   3 +-
 .../google-cloud-platform-core/build.gradle        |   4 +-
 .../extensions/google-cloud-platform-core/pom.xml  |   2 +-
 .../sdk/extensions/gcp/options/GcsOptions.java     |   8 +-
 .../sdk/extensions/gcp/storage/GcsFileSystem.java  |  33 +-
 .../gcp/storage/GcsFileSystemRegistrar.java        |   2 +-
 .../java/org/apache/beam/sdk/util/GcsUtil.java     |  57 +-
 .../beam/sdk/util/RetryHttpRequestInitializer.java |   2 +-
 .../sdk/extensions/gcp/options/GcpOptionsTest.java |   2 +-
 .../extensions/gcp/storage/GcsFileSystemTest.java  |  11 +-
 .../java/org/apache/beam/sdk/util/GcsUtilTest.java |  25 +-
 .../sdk/util/RetryHttpRequestInitializerTest.java  |  30 +-
 sdks/java/extensions/jackson/build.gradle          |   4 +-
 sdks/java/extensions/jackson/pom.xml               |   2 +-
 sdks/java/extensions/join-library/build.gradle     |   4 +-
 sdks/java/extensions/join-library/pom.xml          |   2 +-
 .../beam/sdk/extensions/joinlibrary/Join.java      |  20 +-
 sdks/java/extensions/pom.xml                       |   2 +-
 sdks/java/extensions/protobuf/build.gradle         |   4 +-
 sdks/java/extensions/protobuf/pom.xml              |   2 +-
 .../beam/sdk/extensions/protobuf/ProtoCoder.java   |   2 +-
 .../extensions/protobuf/ByteStringCoderTest.java   |   2 +-
 .../sdk/extensions/protobuf/ProtoCoderTest.java    |   8 +-
 sdks/java/extensions/sketching/pom.xml             |   3 +-
 .../extensions/sketching/ApproximateDistinct.java  |  19 +-
 .../extensions/sketching/SketchFrequencies.java    | 520 +++++++++++
 .../sketching/ApproximateDistinctTest.java         |  13 +-
 .../sketching/SketchFrequenciesTest.java           | 204 +++++
 sdks/java/extensions/sorter/build.gradle           |   4 +-
 sdks/java/extensions/sorter/pom.xml                |   2 +-
 .../beam/sdk/extensions/sorter/InMemorySorter.java |  12 +-
 .../beam/sdk/extensions/sorter/SortValues.java     |   2 +-
 .../sorter/BufferedExternalSorterTest.java         |  21 +-
 .../sdk/extensions/sorter/ExternalSorterTest.java  |  11 +-
 .../sdk/extensions/sorter/InMemorySorterTest.java  |  21 +-
 .../beam/sdk/extensions/sorter/SortValuesTest.java |   5 +-
 sdks/java/extensions/sql/NOTICE                    |   2 +-
 sdks/java/extensions/sql/build.gradle              |  25 +-
 sdks/java/extensions/sql/pom.xml                   |  19 +-
 .../beam/sdk/extensions/sql/BeamRecordSqlType.java |   6 +-
 .../apache/beam/sdk/extensions/sql/BeamSql.java    |   3 +-
 .../sdk/extensions/sql/example/BeamSqlExample.java |  18 +-
 .../operator/BeamSqlCaseExpression.java            |   3 +-
 .../operator/BeamSqlCastExpression.java            |  16 +-
 .../date/BeamSqlCurrentDateExpression.java         |   2 +-
 .../BeamSqlTimestampMinusIntervalExpression.java   |   2 +-
 .../BeamSqlTimestampMinusTimestampExpression.java  |   4 +-
 .../DatetimeReinterpretConversions.java            |  31 +-
 .../reinterpret/IntegerReinterpretConversions.java |  17 +-
 .../operator/reinterpret/Reinterpreter.java        |   4 +-
 .../sql/impl/planner/BeamQueryPlanner.java         |   5 +-
 .../sql/impl/rel/BeamAggregationRel.java           |  14 +-
 .../sdk/extensions/sql/impl/rel/BeamJoinRel.java   |   4 +-
 .../sql/impl/rel/BeamSetOperatorRelBase.java       |  20 +-
 .../sdk/extensions/sql/impl/rel/BeamSortRel.java   |  19 +-
 .../impl/transform/BeamAggregationTransforms.java  |  32 +-
 .../impl/transform/BeamBuiltinAggregations.java    | 279 +-----
 .../sql/impl/transform/BeamJoinTransforms.java     |  10 +-
 .../impl/transform/agg/BigDecimalConverter.java    |  55 ++
 .../impl/transform/agg/VarianceAccumulator.java    |  97 +++
 .../sql/impl/transform/agg/VarianceFn.java         | 138 +++
 .../sql/impl/transform/agg}/package-info.java      |  11 +-
 .../extensions/sql/impl/utils/CalciteUtils.java    |  13 +-
 .../extensions/sql/BeamSqlDslAggregationTest.java  |  20 +-
 .../sql/BeamSqlDslAggregationVarianceTest.java     | 114 +++
 .../beam/sdk/extensions/sql/BeamSqlDslBase.java    |   2 +-
 .../sdk/extensions/sql/BeamSqlDslFilterTest.java   |  12 +-
 .../sdk/extensions/sql/BeamSqlDslJoinTest.java     |  16 +-
 .../sdk/extensions/sql/BeamSqlDslProjectTest.java  |  20 +-
 .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java  |  16 +-
 .../sdk/extensions/sql/BeamSqlNonAsciiTest.java    |  12 +-
 .../impl/interpreter/BeamSqlFnExecutorTest.java    |  43 +-
 .../BeamSqlArithmeticExpressionTest.java           |   4 +-
 .../sql/impl/schema/BeamSqlRowCoderTest.java       |  35 +-
 .../transform/BeamAggregationTransformTest.java    | 366 ++++----
 .../schema/transform/BeamTransformBaseTest.java    |  62 +-
 .../transform/agg/BigDecimalConverterTest.java     |  64 ++
 .../transform/agg/VarianceAccumulatorTest.java     | 132 +++
 .../sql/impl/transform/agg/VarianceFnTest.java     | 104 +++
 .../sql/impl/utils/SqlTypeUtilsTest.java           |  13 +-
 .../BeamSqlMathFunctionsIntegrationTest.java       |   2 +-
 .../meta/provider/kafka/BeamKafkaCSVTableTest.java |  19 +-
 .../meta/provider/text/BeamTextCSVTableTest.java   |  17 +-
 .../extensions/sql/utils/BeamRecordAsserts.java    |  58 ++
 sdks/java/fn-execution/build.gradle                |   9 +-
 sdks/java/fn-execution/pom.xml                     |  12 +-
 .../beam/sdk/fn/channel/SocketAddressFactory.java  |   2 +-
 .../sdk/fn}/data/BeamFnDataGrpcMultiplexer.java    |  74 +-
 .../sdk/fn}/data/BeamFnDataInboundObserver.java    |  51 +-
 .../apache/beam/sdk/fn/data/InboundDataClient.java |  59 ++
 .../beam/sdk/fn/data/RemoteGrpcPortRead.java       |  68 ++
 .../beam/sdk/fn/data/RemoteGrpcPortWrite.java      |  73 ++
 .../fn/data/SettableFutureInboundDataClient.java   |  63 ++
 .../sdk/fn/stream/BufferingStreamObserver.java     |   9 +-
 .../beam/sdk/fn/stream/StreamObserverFactory.java  |   7 +-
 .../sdk/fn/windowing/EncodedBoundedWindow.java     |  94 ++
 .../beam/sdk/fn/windowing}/package-info.java       |   7 +-
 .../BeamFnDataBufferingOutboundObserverTest.java   |  14 +-
 .../fn}/data/BeamFnDataGrpcMultiplexerTest.java    |  42 +-
 .../beam/sdk/fn/data/RemoteGrpcPortReadTest.java   |  75 ++
 .../beam/sdk/fn/data/RemoteGrpcPortWriteTest.java  |  72 ++
 .../beam/sdk/fn/stream/AdvancingPhaserTest.java    |   7 +-
 .../sdk/fn/stream/BufferingStreamObserverTest.java |  62 +-
 .../apache/beam/sdk/fn/stream/DataStreamsTest.java |   9 +-
 .../sdk/fn/stream/DirectStreamObserverTest.java    |  61 +-
 .../ForwardingClientResponseObserverTest.java      |   6 +-
 .../sdk/fn/stream/StreamObserverFactoryTest.java   |   9 +-
 .../org/apache/beam/sdk/fn/test/TestExecutors.java |   7 +-
 .../apache/beam/sdk/fn/test/TestExecutorsTest.java |  21 +-
 .../org/apache/beam/sdk/fn/test/TestStreams.java   |  39 +-
 .../apache/beam/sdk/fn/test/TestStreamsTest.java   |  44 +-
 .../sdk/fn/windowing/EncodedBoundedWindowTest.java |  46 +
 sdks/java/harness/build.gradle                     |  22 +-
 sdks/java/harness/pom.xml                          |  28 +-
 .../beam/fn/harness/BeamFnDataReadRunner.java      |  61 +-
 .../beam/fn/harness/BeamFnDataWriteRunner.java     |  34 +-
 .../org/apache/beam/fn/harness/FlattenRunner.java  |  92 ++
 .../apache/beam/fn/harness/FnApiDoFnRunner.java    | 634 ++++++++------
 .../java/org/apache/beam/fn/harness/FnHarness.java |  14 +
 .../fn/harness/control/ProcessBundleHandler.java   |  67 +-
 .../beam/fn/harness/data/BeamFnDataClient.java     |   7 +-
 .../beam/fn/harness/data/BeamFnDataGrpcClient.java |  41 +-
 .../data/CompletableFutureInboundDataClient.java   |  73 ++
 .../apache/beam/fn/harness/state/BagUserState.java |  66 +-
 .../state/LazyCachingIteratorToIterable.java       |  17 +
 .../beam/fn/harness/state/MultimapSideInput.java   |  85 ++
 .../fn/harness/state/StateFetchingIterators.java   |  28 +-
 .../beam/fn/harness/BeamFnDataReadRunnerTest.java  | 114 ++-
 .../beam/fn/harness/BeamFnDataWriteRunnerTest.java |  84 +-
 .../apache/beam/fn/harness/FlattenRunnerTest.java  | 103 +++
 .../beam/fn/harness/FnApiDoFnRunnerTest.java       | 293 ++++++-
 .../org/apache/beam/fn/harness/FnHarnessTest.java  |  60 +-
 .../harness/control/BeamFnControlClientTest.java   |  41 +-
 .../harness/control/ProcessBundleHandlerTest.java  | 257 +++---
 .../fn/harness/control/RegisterHandlerTest.java    |  17 +-
 .../fn/harness/data/BeamFnDataGrpcClientTest.java  |  43 +-
 .../data/BeamFnDataInboundObserverTest.java        |  31 +-
 .../CompletableFutureInboundDataClientTest.java    | 166 ++++
 .../harness/logging/BeamFnLoggingClientTest.java   |  18 +-
 .../beam/fn/harness/state/BagUserStateTest.java    |  59 +-
 .../state/LazyCachingIteratorToIterableTest.java   |  14 +
 .../fn/harness/state/MultimapSideInputTest.java    |  73 ++
 .../harness/state/StateFetchingIteratorsTest.java  |  49 +-
 .../io/{kafka => amazon-web-services}/build.gradle |  30 +-
 .../java/io/amazon-web-services}/pom.xml           |  81 +-
 .../apache/beam/sdk/io/aws/options/AwsModule.java  | 189 ++++
 .../apache/beam/sdk/io/aws/options/AwsOptions.java |  69 ++
 .../aws/options/AwsPipelineOptionsRegistrar.java}  |  11 +-
 .../apache/beam/sdk/io/aws/options/S3Options.java  |  45 +
 .../beam/sdk/io/aws/options}/package-info.java     |   7 +-
 .../apache/beam/sdk/io/aws/s3/S3FileSystem.java    | 660 ++++++++++++++
 .../beam/sdk/io/aws/s3/S3FileSystemRegistrar.java} |  17 +-
 .../io/aws/s3/S3ReadableSeekableByteChannel.java   | 169 ++++
 .../apache/beam/sdk/io/aws/s3/S3ResourceId.java    | 194 +++++
 .../beam/sdk/io/aws/s3/S3WritableByteChannel.java  | 147 ++++
 .../apache/beam/sdk/io/aws/s3}/package-info.java   |   6 +-
 .../beam/sdk/io/aws/options/AwsModuleTest.java     | 158 ++++
 .../beam/sdk/io/aws/s3/MatchResultMatcher.java     | 118 +++
 .../beam/sdk/io/aws/s3/S3FileSystemTest.java       | 499 +++++++++++
 .../beam/sdk/io/aws/s3/S3ResourceIdTest.java       | 299 +++++++
 .../sdk/io/aws/s3/S3WritableByteChannelTest.java   |  90 ++
 sdks/java/io/amqp/build.gradle                     |   4 +-
 sdks/java/io/amqp/pom.xml                          |   2 +-
 .../java/org/apache/beam/sdk/io/amqp/AmqpIO.java   |   9 +-
 .../org/apache/beam/sdk/io/amqp/AmqpIOTest.java    |   3 +-
 sdks/java/io/cassandra/build.gradle                |   8 +-
 sdks/java/io/cassandra/pom.xml                     |   2 +-
 .../apache/beam/sdk/io/cassandra/CassandraIO.java  |   8 +-
 .../sdk/io/cassandra/CassandraServiceImpl.java     |   6 +-
 .../beam/sdk/io/cassandra/CassandraIOIT.java       |  22 +-
 .../beam/sdk/io/cassandra/CassandraIOTest.java     |  19 +-
 sdks/java/io/common/build.gradle                   |   3 +-
 sdks/java/io/common/pom.xml                        |   7 +-
 .../beam/sdk/io/common/DatabaseTestHelper.java}    |  57 +-
 .../beam/sdk/io/common/IOTestPipelineOptions.java  |   6 +-
 .../org/apache/beam/sdk/io/common/TestRow.java     |   7 +-
 .../elasticsearch-tests-2/build.gradle             |  14 +-
 .../elasticsearch-tests-2/pom.xml                  |   2 +-
 .../elasticsearch-tests-5/build.gradle             |  14 +-
 .../elasticsearch-tests-5/pom.xml                  |  38 +-
 .../elasticsearch-tests-common/build.gradle        |  10 +-
 .../elasticsearch-tests-common/pom.xml             |   2 +-
 .../elasticsearch/ElasticsearchIOTestCommon.java   |  10 +-
 sdks/java/io/elasticsearch-tests/pom.xml           |   2 +-
 sdks/java/io/elasticsearch/build.gradle            |   4 +-
 sdks/java/io/elasticsearch/pom.xml                 |   2 +-
 .../beam/sdk/io/elasticsearch/ElasticsearchIO.java |  36 +-
 sdks/java/io/file-based-io-tests/build.gradle      |  10 +-
 sdks/java/io/file-based-io-tests/pom.xml           | 143 ++-
 .../java/org/apache/beam/sdk/io/avro/AvroIOIT.java |  39 +-
 .../beam/sdk/io/common/FileBasedIOITHelper.java    |  10 +-
 .../java/org/apache/beam/sdk/io/text/TextIOIT.java |  28 +-
 .../apache/beam/sdk/io/tfrecord/TFRecordIOIT.java  |  25 +-
 sdks/java/io/google-cloud-platform/build.gradle    |  14 +-
 sdks/java/io/google-cloud-platform/pom.xml         |   4 +-
 .../beam/sdk/io/gcp/bigquery/BatchLoads.java       | 110 ++-
 .../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java  |   7 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java       |  46 +-
 .../sdk/io/gcp/bigquery/BigQueryQuerySource.java   |  11 +-
 .../sdk/io/gcp/bigquery/BigQueryServicesImpl.java  |  63 +-
 .../sdk/io/gcp/bigquery/BigQuerySourceBase.java    |  11 +-
 .../beam/sdk/io/gcp/bigquery/CreateTables.java     | 169 ++--
 .../sdk/io/gcp/bigquery/DynamicDestinations.java   |  24 +-
 .../gcp/bigquery/DynamicDestinationsHelpers.java   |   4 +-
 .../io/gcp/bigquery/PassThroughThenCleanup.java    |   5 +-
 .../beam/sdk/io/gcp/bigquery/ReifyAsIterable.java  |   2 +-
 .../sdk/io/gcp/bigquery/StreamingWriteTables.java  |  25 +-
 .../beam/sdk/io/gcp/bigquery/WritePartition.java   |   7 +-
 .../beam/sdk/io/gcp/bigquery/WriteResult.java      |   2 +-
 .../beam/sdk/io/gcp/bigquery/WriteTables.java      |  13 +-
 .../beam/sdk/io/gcp/bigtable/BigtableConfig.java   | 100 ++-
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java       | 352 +++++---
 .../sdk/io/gcp/bigtable/BigtableServiceImpl.java   |  61 +-
 .../beam/sdk/io/gcp/datastore/DatastoreV1.java     |   4 +-
 .../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java    |   6 +-
 .../gcp/pubsub/PubsubMessagePayloadOnlyCoder.java  |   3 +-
 .../sdk/io/gcp/pubsub/PubsubUnboundedSink.java     |   2 +-
 .../sdk/io/gcp/pubsub/PubsubUnboundedSource.java   |   4 +-
 .../beam/sdk/io/gcp/spanner/OrderedCode.java       |   3 +-
 .../apache/beam/sdk/io/gcp/spanner/SpannerIO.java  |  41 +-
 .../sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java |  15 +-
 .../sdk/io/gcp/bigquery/BigQueryIOReadTest.java    |   9 +-
 .../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java   | 166 ++--
 .../io/gcp/bigquery/BigQueryServicesImplTest.java  |  10 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java |  25 +-
 .../sdk/io/gcp/bigquery/FakeDatasetService.java    |   7 +-
 .../beam/sdk/io/gcp/bigquery/FakeJobService.java   |   4 +-
 .../sdk/io/gcp/bigtable/BigtableConfigTest.java    | 237 +++++
 .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java   | 273 ++++--
 .../beam/sdk/io/gcp/bigtable/BigtableReadIT.java   |   7 +-
 .../io/gcp/bigtable/BigtableServiceImplTest.java   | 146 ++++
 .../beam/sdk/io/gcp/datastore/DatastoreV1Test.java |  29 +-
 .../apache/beam/sdk/io/gcp/datastore/V1ReadIT.java |   8 +-
 .../sdk/io/gcp/pubsub/PubsubJsonClientTest.java    |   4 +-
 .../sdk/io/gcp/pubsub/PubsubTestClientTest.java    |  13 +-
 .../sdk/io/gcp/pubsub/PubsubUnboundedSinkTest.java |  26 +-
 .../io/gcp/pubsub/PubsubUnboundedSourceTest.java   |   9 +-
 .../io/gcp/spanner/MutationGroupEncoderTest.java   |  14 +-
 .../beam/sdk/io/gcp/spanner/OrderedCodeTest.java   |   4 +-
 .../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java |   6 +-
 .../sdk/io/gcp/spanner/SpannerIOWriteTest.java     |   9 +-
 .../beam/sdk/io/gcp/spanner/SpannerReadIT.java     |   3 +-
 sdks/java/io/hadoop-common/build.gradle            |   2 +-
 sdks/java/io/hadoop-common/pom.xml                 |   2 +-
 sdks/java/io/hadoop-file-system/build.gradle       |   6 +-
 sdks/java/io/hadoop-file-system/pom.xml            |   2 +-
 .../apache/beam/sdk/io/hdfs/HadoopFileSystem.java  |   3 +-
 .../io/hdfs/HadoopFileSystemOptionsRegistrar.java  |   2 +-
 .../sdk/io/hdfs/HadoopFileSystemOptionsTest.java   |   2 +-
 .../beam/sdk/io/hdfs/HadoopFileSystemTest.java     |  31 +-
 .../build.gradle                                   |  33 +-
 sdks/java/io/{jdbc => hadoop-input-format}/pom.xml | 249 ++++--
 .../io/hadoop/inputformat/HadoopInputFormatIO.java |  74 +-
 .../sdk/io/hadoop/inputformat/package-info.java    |   0
 .../ConfigurableEmployeeInputFormat.java           |   2 +-
 .../beam/sdk/io/hadoop/inputformat/Employee.java   |   0
 .../io/hadoop/inputformat/EmployeeInputFormat.java |   2 +-
 .../io/hadoop/inputformat}/HIFIOCassandraIT.java   |  27 +-
 .../sdk/io/hadoop/inputformat}/HIFIOElasticIT.java |  44 +-
 .../hadoop/inputformat/HIFIOWithElasticTest.java   |  26 +-
 .../HIFIOWithEmbeddedCassandraTest.java            |   9 +-
 .../io/hadoop/inputformat/HIFITestOptions.java}    |   4 +-
 .../hadoop/inputformat/HadoopInputFormatIOIT.java  | 151 ++++
 .../inputformat/HadoopInputFormatIOTest.java       |  42 +-
 .../ReuseObjectsEmployeeInputFormat.java           |   2 +-
 .../io/hadoop/inputformat/TestEmployeeDataSet.java |  22 +-
 .../io/hadoop/inputformat/TestRowDBWritable.java   |  83 ++
 .../src/test/resources/cassandra.yaml              |   0
 sdks/java/io/hadoop/input-format/build.gradle      |  47 -
 sdks/java/io/hadoop/input-format/pom.xml           |  78 --
 sdks/java/io/hadoop/jdk1.8-tests/pom.xml           | 273 ------
 sdks/java/io/hadoop/pom.xml                        |  82 --
 sdks/java/io/hbase/build.gradle                    |   8 +-
 sdks/java/io/hbase/pom.xml                         |  28 +-
 .../org/apache/beam/sdk/io/hbase/HBaseIOTest.java  |   8 +-
 .../beam/sdk/io/hbase/SerializableScanTest.java    |   6 +-
 sdks/java/io/hcatalog/build.gradle                 |   6 +-
 sdks/java/io/hcatalog/pom.xml                      |   2 +-
 .../beam/sdk/io/hcatalog/HCatalogIOTest.java       |   2 +-
 .../beam/sdk/io/hcatalog/HCatalogIOTestUtils.java  |   2 +-
 sdks/java/io/jdbc/build.gradle                     |  12 +-
 sdks/java/io/jdbc/pom.xml                          |   9 +-
 .../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java   |  42 +-
 .../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java |  57 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java    | 101 ++-
 .../apache/beam/sdk/io/jdbc/JdbcTestHelper.java    |  35 +-
 sdks/java/io/jms/build.gradle                      |   4 +-
 sdks/java/io/jms/pom.xml                           |   2 +-
 .../java/org/apache/beam/sdk/io/jms/JmsIO.java     | 212 +++--
 .../java/org/apache/beam/sdk/io/jms/JmsIOTest.java |  57 +-
 sdks/java/io/kafka/build.gradle                    |   6 +-
 sdks/java/io/kafka/pom.xml                         |   8 +-
 .../beam/sdk/io/kafka/KafkaCheckpointMark.java     |  20 +-
 .../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 356 ++++----
 .../apache/beam/sdk/io/kafka/KafkaRecordCoder.java |   6 +-
 .../org/apache/beam/sdk/io/kafka/KafkaIOTest.java  | 213 +++--
 sdks/java/io/kinesis/build.gradle                  |   4 +-
 sdks/java/io/kinesis/pom.xml                       |  23 +-
 .../sdk/io/kinesis/DynamicCheckpointGenerator.java |  38 +-
 .../sdk/io/kinesis/GetKinesisRecordsResult.java    |  19 +-
 .../org/apache/beam/sdk/io/kinesis/KinesisIO.java  |  59 +-
 .../apache/beam/sdk/io/kinesis/KinesisReader.java  |  12 +-
 .../apache/beam/sdk/io/kinesis/KinesisRecord.java  |   4 +-
 .../io/kinesis/KinesisShardClosedException.java}   |  13 +-
 .../beam/sdk/io/kinesis/ShardReadersPool.java      | 185 +++-
 .../beam/sdk/io/kinesis/ShardRecordsIterator.java  |  27 +-
 .../sdk/io/kinesis/SimplifiedKinesisClient.java    | 119 ++-
 .../sdk/io/kinesis/StartingPointShardsFinder.java  | 193 +++++
 .../beam/sdk/io/kinesis/AmazonKinesisMock.java     |  46 +-
 .../io/kinesis/DynamicCheckpointGeneratorTest.java |  35 +-
 .../beam/sdk/io/kinesis/KinesisReaderIT.java       |  20 +-
 .../beam/sdk/io/kinesis/KinesisReaderTest.java     |  47 +-
 .../beam/sdk/io/kinesis/KinesisUploader.java       |   4 +-
 .../beam/sdk/io/kinesis/ShardReadersPoolTest.java  | 132 ++-
 .../sdk/io/kinesis/ShardRecordsIteratorTest.java   |  18 +-
 .../io/kinesis/StartingPointShardsFinderTest.java  | 264 ++++++
 sdks/java/io/mongodb/build.gradle                  |   4 +-
 sdks/java/io/mongodb/pom.xml                       |   2 +-
 .../beam/sdk/io/mongodb/MongoDbGridFSIO.java       |  42 +-
 .../beam/sdk/io/mongodb/MongoDBGridFSIOTest.java   | 135 ++-
 .../apache/beam/sdk/io/mongodb/MongoDbIOTest.java  |  85 +-
 sdks/java/io/mqtt/build.gradle                     |   4 +-
 sdks/java/io/mqtt/pom.xml                          |   2 +-
 .../java/org/apache/beam/sdk/io/mqtt/MqttIO.java   |  30 +-
 .../org/apache/beam/sdk/io/mqtt/MqttIOTest.java    | 146 ++--
 sdks/java/io/pom.xml                               |  19 +-
 sdks/java/io/redis/build.gradle                    |   4 +-
 sdks/java/io/redis/pom.xml                         |   2 +-
 .../java/org/apache/beam/sdk/io/redis/RedisIO.java |  13 +-
 .../org/apache/beam/sdk/io/redis/RedisIOTest.java  |   4 +-
 sdks/java/io/solr/build.gradle                     |   6 +-
 sdks/java/io/solr/pom.xml                          |   2 +-
 .../java/org/apache/beam/sdk/io/solr/SolrIO.java   |  11 +-
 .../org/apache/beam/sdk/io/solr/SolrIOTest.java    |   7 +-
 sdks/java/io/tika/build.gradle                     |   8 +-
 sdks/java/io/tika/pom.xml                          |   2 +-
 .../org/apache/beam/sdk/io/tika/TikaIOTest.java    |  14 +-
 sdks/java/io/xml/build.gradle                      |   6 +-
 sdks/java/io/xml/pom.xml                           |   2 +-
 .../java/org/apache/beam/sdk/io/xml/XmlIO.java     |  14 +-
 .../java/org/apache/beam/sdk/io/xml/XmlSource.java |   7 +-
 .../org/apache/beam/sdk/io/xml/JAXBCoderTest.java  |  31 +-
 .../java/org/apache/beam/sdk/io/xml/XmlIOTest.java |   8 +-
 .../org/apache/beam/sdk/io/xml/XmlSourceTest.java  |  11 +-
 sdks/java/java8tests/build.gradle                  |   6 +-
 sdks/java/java8tests/pom.xml                       |  14 +-
 .../options/PipelineOptionsFactoryJava8Test.java   |   2 +-
 .../beam/sdk/transforms/DistinctJava8Test.java     |   8 +-
 .../sdk/transforms/SimpleFunctionJava8Test.java    |   3 +-
 .../beam/sdk/transforms/WithKeysJava8Test.java     |   4 +-
 sdks/java/javadoc/ant.xml                          |   2 +-
 sdks/java/javadoc/pom.xml                          |  12 +-
 .../examples-java8/generate-sources.sh             |  84 --
 sdks/java/maven-archetypes/examples-java8/pom.xml  | 130 ---
 .../META-INF/maven/archetype-metadata.xml          |  39 -
 .../src/main/resources/archetype-resources/pom.xml | 404 ---------
 .../src/test/resources/projects/basic/goal.txt     |   1 -
 sdks/java/maven-archetypes/examples/build.gradle   |   2 +-
 sdks/java/maven-archetypes/examples/pom.xml        |  45 +-
 .../META-INF/maven/archetype-metadata.xml          |   2 +-
 .../src/main/resources/archetype-resources/pom.xml |  35 +-
 .../resources/projects/basic/archetype.properties  |   2 +-
 sdks/java/maven-archetypes/pom.xml                 |  17 +-
 sdks/java/maven-archetypes/starter/build.gradle    |   4 +-
 sdks/java/maven-archetypes/starter/pom.xml         |   2 +-
 .../META-INF/maven/archetype-metadata.xml          |   2 +-
 .../resources/projects/basic/archetype.properties  |   2 +-
 .../resources/projects/basic/reference/pom.xml     |   4 +-
 sdks/java/nexmark/build.gradle                     |  10 +-
 sdks/java/nexmark/pom.xml                          |   5 +-
 .../apache/beam/sdk/nexmark/NexmarkLauncher.java   |  96 ++-
 .../org/apache/beam/sdk/nexmark/model/Bid.java     |  43 +-
 .../model/sql/adapter/ModelAdaptersMapping.java    |  46 +-
 .../beam/sdk/nexmark/queries/NexmarkQuery.java     |  32 +-
 .../apache/beam/sdk/nexmark/queries/Query10.java   | 361 ++++----
 .../apache/beam/sdk/nexmark/queries/Query11.java   |  17 +-
 .../apache/beam/sdk/nexmark/queries/Query12.java   |  49 +-
 .../apache/beam/sdk/nexmark/queries/Query2.java    |  14 +-
 .../apache/beam/sdk/nexmark/queries/Query3.java    |  25 +-
 .../apache/beam/sdk/nexmark/queries/Query4.java    |  26 +-
 .../beam/sdk/nexmark/queries/Query4Model.java      |   9 +-
 .../apache/beam/sdk/nexmark/queries/Query5.java    |   5 +-
 .../beam/sdk/nexmark/queries/Query5Model.java      |   6 +-
 .../apache/beam/sdk/nexmark/queries/Query6.java    |  19 +-
 .../apache/beam/sdk/nexmark/queries/Query7.java    |   9 +-
 .../apache/beam/sdk/nexmark/queries/Query8.java    |  25 +-
 .../beam/sdk/nexmark/queries/WinningBids.java      |  99 ++-
 .../sdk/nexmark/queries/WinningBidsSimulator.java  |   7 +-
 .../beam/sdk/nexmark/queries/sql/SqlQuery1.java    |  81 ++
 .../beam/sdk/nexmark/queries/sql/SqlQuery2.java    |  77 ++
 .../beam/sdk/nexmark/queries/sql/SqlQuery3.java    | 148 ++++
 .../sdk/nexmark/model/sql/BeamRecordSizeTest.java  |  29 +-
 .../sdk/nexmark/queries/sql/SqlQuery1Test.java     |  84 ++
 .../sdk/nexmark/queries/sql/SqlQuery2Test.java     | 113 +++
 .../sdk/nexmark/queries/sql/SqlQuery3Test.java     | 147 ++++
 sdks/java/pom.xml                                  |  19 +-
 sdks/pom.xml                                       |   2 +-
 .../apache_beam/examples/snippets/snippets_test.py |  31 +-
 .../python/apache_beam/examples/wordcount_fnapi.py |   8 +-
 sdks/python/apache_beam/io/filesystem.py           |   6 +
 sdks/python/apache_beam/io/filesystems.py          |  13 +-
 sdks/python/apache_beam/io/filesystems_test.py     |  20 +-
 sdks/python/apache_beam/io/gcp/bigquery_test.py    |  58 +-
 .../apache_beam/io/gcp/gcsfilesystem_test.py       |  91 +-
 sdks/python/apache_beam/io/gcp/tests/utils_test.py |  21 +-
 sdks/python/apache_beam/io/hadoopfilesystem.py     |   4 +-
 .../python/apache_beam/io/hadoopfilesystem_test.py |   4 +-
 sdks/python/apache_beam/io/localfilesystem_test.py |  24 +-
 sdks/python/apache_beam/io/tfrecordio_test.py      |   4 +-
 sdks/python/apache_beam/io/vcfio.py                |  22 +-
 sdks/python/apache_beam/metrics/execution.py       |   4 +-
 sdks/python/apache_beam/pipeline.py                |  74 +-
 sdks/python/apache_beam/pipeline_test.py           |  29 +-
 sdks/python/apache_beam/pvalue.py                  |  10 +-
 sdks/python/apache_beam/runners/__init__.py        |   1 -
 .../runners/dataflow/dataflow_runner.py            |   9 +
 .../runners/dataflow/internal/apiclient.py         |   2 +-
 .../runners/dataflow/native_io/iobase_test.py      |   3 +-
 .../apache_beam/runners/direct/direct_runner.py    |  55 --
 .../runners/direct/direct_runner_test.py           |   2 +-
 .../runners/direct/evaluation_context.py           |  14 -
 sdks/python/apache_beam/runners/direct/executor.py |  11 -
 .../runners/direct/sdf_direct_runner_test.py       |   6 +-
 .../runners/portability/fn_api_runner.py           |   7 +-
 .../runners/portability/fn_api_runner_test.py      |  10 +-
 .../runners/portability/maptask_executor_runner.py |   6 +-
 .../runners/portability/universal_local_runner.py  |   1 +
 sdks/python/apache_beam/runners/runner.py          |   2 +-
 .../apache_beam/runners/worker/bundle_processor.py |  25 +-
 .../apache_beam/runners/worker/opcounters.pxd      |  17 +
 .../apache_beam/runners/worker/opcounters.py       | 108 +++
 .../apache_beam/runners/worker/opcounters_test.py  |  46 +
 .../apache_beam/runners/worker/operations.py       |  41 +-
 .../apache_beam/runners/worker/sdk_worker.py       |   9 +-
 .../apache_beam/runners/worker/sideinputs.py       |  49 +-
 .../apache_beam/runners/worker/sideinputs_test.py  |  47 +-
 .../apache_beam/runners/worker/statesampler.py     |  81 ++
 .../runners/worker/statesampler_fake.py            |  51 --
 .../{statesampler.pyx => statesampler_fast.pyx}    | 137 +--
 .../runners/worker/statesampler_slow.py            |  76 ++
 .../runners/worker/statesampler_test.py            |  48 +-
 sdks/python/apache_beam/testing/util.py            |   3 +
 sdks/python/apache_beam/transforms/core.py         |  14 +-
 sdks/python/apache_beam/transforms/ptransform.py   | 119 ++-
 .../apache_beam/transforms/ptransform_test.py      |  38 +-
 sdks/python/apache_beam/transforms/util.py         |  11 +-
 sdks/python/apache_beam/transforms/util_test.py    |  50 +-
 sdks/python/apache_beam/utils/counters.py          |  36 +-
 sdks/python/apache_beam/version.py                 |   2 +-
 sdks/python/build.gradle                           |   6 +-
 sdks/python/container/build.gradle                 |   4 +-
 sdks/python/container/pom.xml                      |   2 +-
 sdks/python/gen_protos.py                          |   9 +
 sdks/python/pom.xml                                |   8 +-
 sdks/python/setup.cfg                              |  29 +
 sdks/python/tox.ini                                |  21 +
 settings.gradle                                    | 218 ++---
 1092 files changed, 26294 insertions(+), 17559 deletions(-)

-- 
To stop receiving notification emails like this one, please contact
lcwik@apache.org.

Mime
View raw message