beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [49/50] [abbrv] beam git commit: Merge branch 'master' upto commit 686b774ceda8bee32032cb421651e8350ca5bf3d into jstorm-runner
Date Tue, 25 Apr 2017 17:30:44 GMT
Merge branch 'master' upto commit 686b774ceda8bee32032cb421651e8350ca5bf3d into jstorm-runner


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f1e170a5
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f1e170a5
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f1e170a5

Branch: refs/heads/jstorm-runner
Commit: f1e170a5fa9dc4d462af42f9f382afd0ecd798b6
Parents: f6a89b0 686b774
Author: Pei He <hepei.hp@alibaba-inc.com>
Authored: Tue Apr 25 17:37:52 2017 +0800
Committer: Pei He <hepei.hp@alibaba-inc.com>
Committed: Tue Apr 25 18:19:41 2017 +0800

----------------------------------------------------------------------
 .gitignore                                      |    3 +
 .jenkins/common_job_properties.groovy           |  261 ----
 .../job_beam_PerformanceTests_Dataflow.groovy   |   43 -
 .jenkins/job_beam_PerformanceTests_JDBC.groovy  |   60 -
 .jenkins/job_beam_PerformanceTests_Spark.groovy |   44 -
 ...job_beam_PostCommit_Java_MavenInstall.groovy |   42 -
 ..._PostCommit_Java_ValidatesRunner_Apex.groovy |   48 -
 ...tCommit_Java_ValidatesRunner_Dataflow.groovy |   45 -
 ...PostCommit_Java_ValidatesRunner_Flink.groovy |   43 -
 ...tCommit_Java_ValidatesRunner_Gearpump.groovy |   49 -
 ...PostCommit_Java_ValidatesRunner_Spark.groovy |   44 -
 .../job_beam_PostCommit_Python_Verify.groovy    |   55 -
 .../job_beam_PreCommit_Java_MavenInstall.groovy |   42 -
 .../job_beam_PreCommit_Website_Stage.groovy     |   80 -
 .jenkins/job_beam_PreCommit_Website_Test.groovy |   65 -
 .../job_beam_Release_NightlySnapshot.groovy     |   45 -
 .jenkins/job_seed.groovy                        |   53 -
 .../jenkins/common_job_properties.groovy        |  261 ++++
 .../job_beam_PerformanceTests_Dataflow.groovy   |   43 +
 .../job_beam_PerformanceTests_JDBC.groovy       |   60 +
 .../job_beam_PerformanceTests_Spark.groovy      |   44 +
 ...job_beam_PostCommit_Java_MavenInstall.groovy |   42 +
 ..._PostCommit_Java_ValidatesRunner_Apex.groovy |   48 +
 ...tCommit_Java_ValidatesRunner_Dataflow.groovy |   45 +
 ...PostCommit_Java_ValidatesRunner_Flink.groovy |   43 +
 ...tCommit_Java_ValidatesRunner_Gearpump.groovy |   49 +
 ...PostCommit_Java_ValidatesRunner_Spark.groovy |   44 +
 .../job_beam_PostCommit_Python_Verify.groovy    |   55 +
 .../job_beam_PreCommit_Java_MavenInstall.groovy |   42 +
 .../job_beam_PreCommit_Website_Stage.groovy     |   80 +
 .../job_beam_PreCommit_Website_Test.groovy      |   65 +
 .../job_beam_Release_NightlySnapshot.groovy     |   45 +
 .test-infra/jenkins/job_seed.groovy             |   53 +
 .../cassandra-service-for-local-dev.yaml        |   28 +
 .../cassandra-svc-statefulset.yaml              |  114 ++
 .../LargeITCluster/cassandra-svc-temp.yaml      |   74 +
 .../cassandra/LargeITCluster/data-load.sh       |  122 ++
 .../cassandra/LargeITCluster/show_health.sh     |   47 +
 .../cassandra/LargeITCluster/start-up.sh        |   22 +
 .../cassandra/LargeITCluster/teardown.sh        |   25 +
 .../cassandra-service-for-local-dev.yaml        |   30 +
 .../SmallITCluster/cassandra-svc-rc.yaml        |   74 +
 .../cassandra/SmallITCluster/data-load.sh       |   86 +
 .../cassandra/SmallITCluster/show_health.sh     |   47 +
 .../cassandra/SmallITCluster/start-up.sh        |   23 +
 .../cassandra/SmallITCluster/teardown.sh        |   22 +
 .../kubernetes/cassandra/data-load-setup.sh     |   29 +
 .../elasticsearch-service-for-local-dev.yaml    |   33 +
 .../es-services-deployments.yaml                |  258 +++
 .../LargeProductionCluster/start-up.sh          |   22 +
 .../LargeProductionCluster/teardown.sh          |   21 +
 .../elasticsearch-service-for-local-dev.yaml    |   34 +
 .../SmallITCluster/elasticsearch-svc-rc.yaml    |   96 ++
 .../elasticsearch/SmallITCluster/start-up.sh    |   23 +
 .../elasticsearch/SmallITCluster/teardown.sh    |   21 +
 .../kubernetes/elasticsearch/data-load-setup.sh |   26 +
 .../kubernetes/elasticsearch/data-load.sh       |   33 +
 .../kubernetes/elasticsearch/es_test_data.py    |  299 ++++
 .../kubernetes/elasticsearch/show-health.sh     |   33 +
 .../postgres/postgres-service-for-local-dev.yml |   28 +
 .test-infra/kubernetes/postgres/postgres.yml    |   56 +
 .test-infra/travis/README.md                    |   23 +
 .test-infra/travis/settings.xml                 |   33 +
 .test-infra/travis/test_wordcount.sh            |  125 ++
 .travis.yml                                     |    4 +-
 .travis/README.md                               |   23 -
 .travis/settings.xml                            |   33 -
 .travis/test_wordcount.sh                       |  125 --
 README.md                                       |    1 +
 examples/java8/pom.xml                          |   34 +
 .../beam/examples/MinimalWordCountJava8.java    |    9 +-
 .../beam/examples/complete/game/GameStats.java  |    8 +-
 .../examples/complete/game/LeaderBoard.java     |    2 +-
 .../beam/examples/complete/game/UserScore.java  |    5 +-
 .../examples/MinimalWordCountJava8Test.java     |    9 +-
 .../complete/game/HourlyTeamScoreTest.java      |    5 +-
 .../examples/complete/game/UserScoreTest.java   |    6 +-
 .../apache/beam/runners/apex/ApexRunner.java    |   34 +-
 .../beam/runners/apex/ApexYarnLauncher.java     |  111 +-
 .../translation/ApexPipelineTranslator.java     |    2 +-
 .../FlattenPCollectionTranslator.java           |   13 +-
 .../apex/translation/GroupByKeyTranslator.java  |    4 +-
 .../apex/translation/ParDoTranslator.java       |   30 +-
 .../apex/translation/TranslationContext.java    |   22 +-
 .../translation/WindowAssignTranslator.java     |   58 +-
 .../operators/ApexGroupByKeyOperator.java       |  273 +---
 .../operators/ApexParDoOperator.java            |  235 ++-
 .../operators/ApexProcessFnOperator.java        |  184 +++
 .../translation/utils/ApexStateInternals.java   |   73 +-
 .../apex/translation/utils/NoOpStepContext.java |    2 +-
 .../utils/SerializablePipelineOptions.java      |   13 +-
 .../translation/utils/StateInternalsProxy.java  |   67 +
 .../translation/ApexGroupByKeyOperatorTest.java |    2 +-
 .../apex/translation/ParDoTranslatorTest.java   |   20 +-
 .../utils/ApexStateInternalsTest.java           |   25 +-
 runners/core-construction-java/pom.xml          |   36 +
 .../beam/runners/core/construction/Coders.java  |  174 +++
 .../DeduplicatedFlattenFactory.java             |   79 +-
 .../EmptyFlattenAsCreateFactory.java            |   25 +-
 .../runners/core/construction/PCollections.java |   97 ++
 .../core/construction/PTransformMatchers.java   |    7 +-
 .../construction/PTransformReplacements.java    |   69 +
 .../core/construction/PrimitiveCreate.java      |   18 +-
 .../core/construction/ReplacementOutputs.java   |   63 +-
 .../core/construction/SdkComponents.java        |  159 ++
 .../SingleInputOutputOverrideFactory.java       |   14 +-
 .../runners/core/construction/Triggers.java     |  336 ++++
 .../UnboundedReadFromBoundedSource.java         |  542 +++++++
 .../UnsupportedOverrideFactory.java             |   20 +-
 .../core/construction/WindowingStrategies.java  |  245 +++
 .../runners/core/construction/CodersTest.java   |  163 ++
 .../DeduplicatedFlattenFactoryTest.java         |   24 +-
 .../EmptyFlattenAsCreateFactoryTest.java        |   42 +-
 .../core/construction/PCollectionsTest.java     |  188 +++
 .../construction/PTransformMatchersTest.java    |  132 +-
 .../PTransformReplacementsTest.java             |  131 ++
 .../construction/ReplacementOutputsTest.java    |  109 +-
 .../core/construction/SdkComponentsTest.java    |  157 ++
 .../SingleInputOutputOverrideFactoryTest.java   |   37 +-
 .../runners/core/construction/TriggersTest.java |  111 ++
 .../UnboundedReadFromBoundedSourceTest.java     |  373 +++++
 .../UnsupportedOverrideFactoryTest.java         |   16 +-
 .../construction/WindowingStrategiesTest.java   |  110 ++
 runners/core-java/pom.xml                       |   10 +-
 .../beam/runners/core/AssignWindowsDoFn.java    |   78 -
 .../beam/runners/core/BaseExecutionContext.java |   13 +-
 .../apache/beam/runners/core/DoFnAdapters.java  |  323 ----
 .../apache/beam/runners/core/DoFnRunners.java   |   10 +-
 .../beam/runners/core/ExecutionContext.java     |   13 +-
 .../GroupAlsoByWindowViaOutputBufferDoFn.java   |   19 +-
 .../core/GroupAlsoByWindowViaWindowSetDoFn.java |    9 +-
 .../GroupAlsoByWindowViaWindowSetNewDoFn.java   |   19 +-
 .../core/GroupAlsoByWindowsAggregators.java     |   28 +
 .../runners/core/GroupAlsoByWindowsDoFn.java    |   46 -
 .../core/LateDataDroppingDoFnRunner.java        |    3 +-
 .../org/apache/beam/runners/core/OldDoFn.java   |   41 +-
 ...eBoundedSplittableProcessElementInvoker.java |  133 +-
 .../beam/runners/core/OutputWindowedValue.java  |   10 +-
 .../beam/runners/core/SimpleDoFnRunner.java     |   59 +-
 .../beam/runners/core/SimpleOldDoFnRunner.java  |   63 +-
 .../beam/runners/core/SplittableParDo.java      |   36 +-
 .../core/SplittableProcessElementInvoker.java   |   22 +-
 .../core/UnboundedReadFromBoundedSource.java    |  542 -------
 .../beam/runners/core/WindowingInternals.java   |    8 +-
 .../core/WindowingInternalsAdapters.java        |    8 +-
 .../triggers/AfterWatermarkStateMachine.java    |   14 +-
 ...roupAlsoByWindowViaOutputBufferDoFnTest.java |    4 +-
 .../core/GroupAlsoByWindowsProperties.java      |   37 +-
 .../apache/beam/runners/core/NoOpOldDoFn.java   |    4 +-
 .../apache/beam/runners/core/OldDoFnTest.java   |    4 +-
 ...ndedSplittableProcessElementInvokerTest.java |   27 +-
 .../beam/runners/core/ReduceFnTester.java       |   13 +-
 .../runners/core/SimpleOldDoFnRunnerTest.java   |    4 +-
 .../beam/runners/core/SplittableParDoTest.java  |  238 +--
 .../UnboundedReadFromBoundedSourceTest.java     |  373 -----
 .../direct/BoundedReadEvaluatorFactory.java     |    2 +-
 ...ectGBKIntoKeyedWorkItemsOverrideFactory.java |   16 +-
 .../beam/runners/direct/DirectGraphVisitor.java |    5 +-
 .../direct/DirectGroupByKeyOverrideFactory.java |   14 +-
 .../direct/ExecutorServiceParallelExecutor.java |    4 +-
 .../runners/direct/FlattenEvaluatorFactory.java |    2 +-
 .../GroupAlsoByWindowEvaluatorFactory.java      |   18 +-
 .../direct/GroupByKeyOnlyEvaluatorFactory.java  |    4 +-
 .../direct/KeyedPValueTrackingVisitor.java      |   14 +-
 .../beam/runners/direct/ModelEnforcement.java   |   13 +-
 .../beam/runners/direct/ParDoEvaluator.java     |    4 +-
 .../runners/direct/ParDoEvaluatorFactory.java   |   20 +-
 .../direct/ParDoMultiOverrideFactory.java       |   29 +-
 ...littableProcessElementsEvaluatorFactory.java |    8 +-
 .../direct/StatefulParDoEvaluatorFactory.java   |   10 +-
 .../direct/TestStreamEvaluatorFactory.java      |   20 +-
 .../direct/TransformEvaluatorFactory.java       |   10 +-
 .../direct/TransformExecutorServices.java       |   37 +-
 .../direct/UnboundedReadEvaluatorFactory.java   |    4 +-
 .../runners/direct/ViewEvaluatorFactory.java    |    4 +-
 .../runners/direct/ViewOverrideFactory.java     |   23 +-
 .../beam/runners/direct/WatermarkManager.java   |   19 +-
 .../runners/direct/WindowEvaluatorFactory.java  |    2 +-
 .../direct/WriteWithShardingFactory.java        |   22 +-
 .../runners/direct/DirectGraphVisitorTest.java  |    7 +-
 .../DirectGroupByKeyOverrideFactoryTest.java    |   12 +-
 .../beam/runners/direct/DirectRunnerTest.java   |    5 +-
 .../beam/runners/direct/ParDoEvaluatorTest.java |    8 +-
 .../direct/ParDoMultiOverrideFactoryTest.java   |   45 -
 .../StatefulParDoEvaluatorFactoryTest.java      |    2 +-
 .../direct/TestStreamEvaluatorFactoryTest.java  |   11 -
 .../direct/TransformExecutorServicesTest.java   |   48 +
 .../runners/direct/ViewOverrideFactoryTest.java |   44 +-
 .../direct/WriteWithShardingFactoryTest.java    |   23 +-
 .../examples/streaming/KafkaIOExamples.java     |  338 ----
 .../KafkaWindowedWordCountExample.java          |  164 --
 runners/flink/pom.xml                           |   45 +-
 runners/flink/runner/pom.xml                    |  109 +-
 .../flink/FlinkBatchTransformTranslators.java   |   36 +-
 .../flink/FlinkBatchTranslationContext.java     |   11 +-
 .../flink/FlinkDetachedRunnerResult.java        |    3 +-
 .../flink/FlinkStreamingPipelineTranslator.java |   61 +-
 .../FlinkStreamingTransformTranslators.java     |  139 +-
 .../flink/FlinkStreamingTranslationContext.java |   12 +-
 .../functions/FlinkDoFnFunction.java            |    4 +-
 .../functions/FlinkNoOpStepContext.java         |    2 +-
 .../functions/FlinkStatefulDoFnFunction.java    |    4 +-
 .../types/EncodedValueTypeInformation.java      |    9 -
 .../flink/translation/types/FlinkCoder.java     |   63 -
 .../utils/SerializedPipelineOptions.java        |    2 +
 .../wrappers/streaming/DoFnOperator.java        |   14 +-
 .../streaming/SingletonKeyedWorkItem.java       |    2 -
 .../streaming/SingletonKeyedWorkItemCoder.java  |    4 +-
 .../streaming/SplittableDoFnOperator.java       |   10 +-
 .../wrappers/streaming/WindowDoFnOperator.java  |    7 +-
 .../streaming/io/UnboundedFlinkSink.java        |  200 ---
 .../streaming/io/UnboundedFlinkSource.java      |  120 --
 .../beam/runners/flink/PipelineOptionsTest.java |    2 +-
 .../flink/streaming/DoFnOperatorTest.java       |   34 +-
 runners/google-cloud-dataflow-java/pom.xml      |   25 +-
 .../dataflow/BatchStatefulParDoOverrides.java   |   51 +-
 .../runners/dataflow/BatchViewOverrides.java    |   23 +-
 .../dataflow/DataflowPipelineTranslator.java    |   37 +-
 .../beam/runners/dataflow/DataflowRunner.java   |  146 +-
 .../DataflowUnboundedReadFromBoundedSource.java |  547 -------
 .../dataflow/PrimitiveParDoSingleFactory.java   |   15 +-
 .../dataflow/ReshuffleOverrideFactory.java      |   12 +-
 .../dataflow/StreamingViewOverrides.java        |   14 +-
 .../runners/dataflow/TransformTranslator.java   |    6 +-
 .../dataflow/testing/TestDataflowRunner.java    |   34 +-
 .../dataflow/BatchViewOverridesTest.java        |    4 +-
 .../dataflow/DataflowPipelineJobTest.java       |    7 +-
 ...aflowUnboundedReadFromBoundedSourceTest.java |   79 -
 .../PrimitiveParDoSingleFactoryTest.java        |   59 +-
 .../testing/TestDataflowRunnerTest.java         |    3 +-
 .../apache/beam/runners/spark/SparkRunner.java  |   20 +-
 .../beam/runners/spark/TestSparkRunner.java     |   21 +-
 .../beam/runners/spark/io/MicrobatchSource.java |  113 +-
 .../beam/runners/spark/io/SourceDStream.java    |   11 +-
 .../SparkGroupAlsoByWindowViaWindowSet.java     |   22 +-
 .../spark/stateful/StateSpecFunctions.java      |    6 +-
 .../runners/spark/translation/DoFnFunction.java |  130 --
 .../spark/translation/EvaluationContext.java    |   11 +-
 .../spark/translation/MultiDoFnFunction.java    |    4 +-
 .../spark/translation/SparkAssignWindowFn.java  |    3 +-
 ...SparkGroupAlsoByWindowViaOutputBufferFn.java |   18 +-
 .../spark/translation/SparkProcessContext.java  |    2 +-
 .../spark/translation/SparkRuntimeContext.java  |    2 +
 .../spark/translation/TransformTranslator.java  |   93 +-
 .../streaming/StreamingTransformTranslator.java |   96 +-
 .../spark/util/SparkSideInputReader.java        |    3 +-
 .../ResumeFromCheckpointStreamingTest.java      |   14 +-
 sdks/common/fn-api/pom.xml                      |    5 -
 .../fn-api/src/main/proto/beam_fn_api.proto     |  174 +--
 .../src/main/proto/beam_runner_api.proto        |   14 +-
 sdks/java/core/pom.xml                          |   44 -
 .../main/java/org/apache/beam/sdk/Pipeline.java |   39 +-
 .../java/org/apache/beam/sdk/io/AvroIO.java     |   13 +-
 .../java/org/apache/beam/sdk/io/PubsubIO.java   | 1195 --------------
 .../apache/beam/sdk/io/PubsubUnboundedSink.java |  494 ------
 .../beam/sdk/io/PubsubUnboundedSource.java      | 1463 ------------------
 .../main/java/org/apache/beam/sdk/io/Sink.java  |    2 +-
 .../org/apache/beam/sdk/io/UnboundedSource.java |   13 +
 .../beam/sdk/options/PipelineOptions.java       |   12 +-
 .../sdk/runners/PTransformOverrideFactory.java  |   31 +-
 .../apache/beam/sdk/runners/PipelineRunner.java |    7 +-
 .../beam/sdk/runners/TransformHierarchy.java    |   80 +-
 .../apache/beam/sdk/testing/TestPipeline.java   |    2 +
 .../beam/sdk/transforms/AppliedPTransform.java  |   24 +-
 .../org/apache/beam/sdk/transforms/Combine.java |    4 +-
 .../org/apache/beam/sdk/transforms/Create.java  |    7 +-
 .../org/apache/beam/sdk/transforms/DoFn.java    |   96 +-
 .../apache/beam/sdk/transforms/DoFnTester.java  |   46 +-
 .../beam/sdk/transforms/FlatMapElements.java    |  113 +-
 .../apache/beam/sdk/transforms/GroupByKey.java  |    4 +-
 .../apache/beam/sdk/transforms/MapElements.java |   99 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |   82 +-
 .../apache/beam/sdk/transforms/Partition.java   |    2 +-
 .../org/apache/beam/sdk/transforms/Sample.java  |    4 +
 .../transforms/join/KeyedPCollectionTuple.java  |   12 +-
 .../reflect/ByteBuddyDoFnInvokerFactory.java    |   47 +-
 .../sdk/transforms/reflect/DoFnInvoker.java     |    4 +-
 .../sdk/transforms/reflect/DoFnSignature.java   |   10 +-
 .../sdk/transforms/reflect/DoFnSignatures.java  |   96 +-
 .../splittabledofn/HasDefaultTracker.java       |   30 +
 .../transforms/splittabledofn/OffsetRange.java  |    8 +-
 .../splittabledofn/OffsetRangeTracker.java      |   33 +-
 .../splittabledofn/RestrictionTracker.java      |    8 +
 .../transforms/windowing/AfterWatermark.java    |   14 +-
 .../beam/sdk/transforms/windowing/Triggers.java |  320 ----
 .../beam/sdk/transforms/windowing/WindowFn.java |   13 -
 .../org/apache/beam/sdk/util/PubsubClient.java  |  544 -------
 .../apache/beam/sdk/util/PubsubGrpcClient.java  |  424 -----
 .../apache/beam/sdk/util/PubsubJsonClient.java  |  317 ----
 .../apache/beam/sdk/util/PubsubTestClient.java  |  436 ------
 .../org/apache/beam/sdk/util/Transport.java     |    3 +-
 .../beam/sdk/util/WindowingStrategies.java      |  267 ----
 .../java/org/apache/beam/sdk/values/PBegin.java |    6 +-
 .../apache/beam/sdk/values/PCollectionList.java |   27 +-
 .../beam/sdk/values/PCollectionTuple.java       |   13 +-
 .../java/org/apache/beam/sdk/values/PDone.java  |    6 +-
 .../java/org/apache/beam/sdk/values/PInput.java |    4 +-
 .../org/apache/beam/sdk/values/POutput.java     |    4 +-
 .../java/org/apache/beam/sdk/values/PValue.java |    4 +-
 .../org/apache/beam/sdk/values/PValueBase.java  |    6 +-
 .../apache/beam/sdk/values/TaggedPValue.java    |    5 +
 .../org/apache/beam/sdk/values/TupleTag.java    |   26 +-
 .../apache/beam/sdk/values/TupleTagList.java    |    2 +-
 .../apache/beam/sdk/values/TypeDescriptors.java |   25 +-
 .../org/apache/beam/sdk/values/TypedPValue.java |    4 +-
 .../java/org/apache/beam/sdk/PipelineTest.java  |  108 +-
 .../org/apache/beam/sdk/io/PubsubIOTest.java    |  197 ---
 .../beam/sdk/io/PubsubUnboundedSinkTest.java    |  190 ---
 .../beam/sdk/io/PubsubUnboundedSourceTest.java  |  411 -----
 .../apache/beam/sdk/metrics/MetricsTest.java    |    2 +-
 .../sdk/runners/TransformHierarchyTest.java     |   72 +-
 .../beam/sdk/testing/StaticWindowsTest.java     |   10 +-
 .../apache/beam/sdk/transforms/CreateTest.java  |    8 +-
 .../beam/sdk/transforms/MapElementsTest.java    |   25 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   |  293 ++--
 .../beam/sdk/transforms/SplittableDoFnTest.java |   58 +-
 .../transforms/reflect/DoFnInvokersTest.java    |  153 +-
 .../DoFnSignaturesProcessElementTest.java       |    2 +-
 .../DoFnSignaturesSplittableDoFnTest.java       |  117 +-
 .../splittabledofn/OffsetRangeTrackerTest.java  |   49 +-
 .../sdk/transforms/windowing/TriggersTest.java  |  100 --
 .../apache/beam/sdk/util/PubsubClientTest.java  |  189 ---
 .../beam/sdk/util/PubsubGrpcClientTest.java     |  207 ---
 .../beam/sdk/util/PubsubJsonClientTest.java     |  140 --
 .../beam/sdk/util/PubsubTestClientTest.java     |  114 --
 .../beam/sdk/util/WindowingStrategiesTest.java  |   91 --
 .../beam/sdk/values/PCollectionListTest.java    |   70 +-
 .../beam/sdk/values/PCollectionTupleTest.java   |   13 +-
 .../apache/beam/sdk/values/TypedPValueTest.java |   46 +-
 .../beam/fn/harness/fake/FakeStepContext.java   |    2 +-
 .../control/ProcessBundleHandlerTest.java       |   30 +-
 sdks/java/io/common/pom.xml                     |    4 +
 .../apache/beam/sdk/io/common/HashingFn.java    |  109 ++
 .../sdk/io/elasticsearch/ElasticsearchIO.java   |    2 +
 sdks/java/io/google-cloud-platform/pom.xml      |   75 +-
 .../sdk/io/gcp/bigquery/BatchLoadBigQuery.java  |    7 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    |   59 +-
 .../io/gcp/bigquery/BigQueryTableSource.java    |   30 +-
 .../sdk/io/gcp/bigquery/WritePartition.java     |    6 +-
 .../beam/sdk/io/gcp/bigquery/WriteResult.java   |   11 +-
 .../beam/sdk/io/gcp/pubsub/PubsubClient.java    |  544 +++++++
 .../sdk/io/gcp/pubsub/PubsubGrpcClient.java     |  424 +++++
 .../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 1014 ++++++++++++
 .../sdk/io/gcp/pubsub/PubsubJsonClient.java     |  319 ++++
 .../sdk/io/gcp/pubsub/PubsubTestClient.java     |  436 ++++++
 .../sdk/io/gcp/pubsub/PubsubUnboundedSink.java  |  490 ++++++
 .../io/gcp/pubsub/PubsubUnboundedSource.java    | 1463 ++++++++++++++++++
 .../beam/sdk/io/gcp/pubsub/package-info.java    |   24 +
 .../beam/sdk/io/gcp/GcpApiSurfaceTest.java      |    5 +-
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     |    4 +-
 .../sdk/io/gcp/pubsub/PubsubClientTest.java     |  189 +++
 .../sdk/io/gcp/pubsub/PubsubGrpcClientTest.java |  208 +++
 .../beam/sdk/io/gcp/pubsub/PubsubIOTest.java    |  189 +++
 .../sdk/io/gcp/pubsub/PubsubJsonClientTest.java |  139 ++
 .../sdk/io/gcp/pubsub/PubsubTestClientTest.java |  114 ++
 .../io/gcp/pubsub/PubsubUnboundedSinkTest.java  |  188 +++
 .../gcp/pubsub/PubsubUnboundedSourceTest.java   |  409 +++++
 .../hadoop/inputformat/HadoopInputFormatIO.java |   36 +-
 .../inputformat/HadoopInputFormatIOTest.java    |   99 +-
 sdks/java/io/hadoop/jdk1.8-tests/pom.xml        |   46 +-
 .../inputformat/HIFIOWithElasticTest.java       |    6 +-
 .../HIFIOWithEmbeddedCassandraTest.java         |  215 +++
 .../hadoop/inputformat/hashing/HashingFn.java   |  109 --
 .../integration/tests/HIFIOCassandraIT.java     |    8 +-
 .../integration/tests/HIFIOElasticIT.java       |    6 +-
 .../SmallITCluster/cassandra-svc-rc.yaml        |   88 --
 .../cassandra/SmallITCluster/start-up.sh        |   21 -
 .../cassandra/SmallITCluster/teardown.sh        |   21 -
 .../kubernetes/cassandra/data-load-setup.sh     |   29 -
 .../resources/kubernetes/cassandra/data-load.sh |   67 -
 .../LargeProductionCluster/es-services.yaml     |  277 ----
 .../LargeProductionCluster/start-up.sh          |   21 -
 .../LargeProductionCluster/teardown.sh          |   20 -
 .../SmallITCluster/elasticsearch-svc-rc.yaml    |   84 -
 .../elasticsearch/SmallITCluster/start-up.sh    |   22 -
 .../elasticsearch/SmallITCluster/teardown.sh    |   20 -
 .../kubernetes/elasticsearch/data-load-setup.sh |   26 -
 .../kubernetes/elasticsearch/data-load.sh       |   33 -
 .../kubernetes/elasticsearch/es_test_data.py    |  299 ----
 .../kubernetes/elasticsearch/show-health.sh     |   25 -
 sdks/java/io/hadoop/pom.xml                     |   12 +-
 .../apache/beam/sdk/io/hdfs/HDFSFileSink.java   |    2 +
 .../apache/beam/sdk/io/hdfs/HDFSFileSource.java |    2 +
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java     |   61 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java |   10 +-
 .../kubernetes/postgres-pod-no-vol.yml          |   32 -
 .../kubernetes/postgres-service-public.yml      |   28 -
 .../jdbc/src/test/resources/kubernetes/setup.sh |   20 -
 .../src/test/resources/kubernetes/teardown.sh   |   20 -
 .../java/org/apache/beam/sdk/io/jms/JmsIO.java  |    2 +
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java   |    2 +
 .../apache/beam/sdk/io/kinesis/KinesisIO.java   |    2 +
 .../beam/sdk/io/mongodb/MongoDbGridFSIO.java    |    9 +-
 .../apache/beam/sdk/io/mongodb/MongoDbIO.java   |    2 +
 .../org/apache/beam/sdk/io/mqtt/MqttIO.java     |    2 +
 .../transforms/FlatMapElementsJava8Test.java    |   10 +-
 .../sdk/transforms/MapElementsJava8Test.java    |   10 +-
 sdks/python/.pylintrc                           |    6 +-
 sdks/python/apache_beam/coders/coder_impl.py    |   50 +-
 sdks/python/apache_beam/coders/coders.py        |    6 +-
 .../examples/complete/game/user_score.py        |    8 +-
 .../examples/complete/top_wikipedia_sessions.py |    8 -
 .../examples/cookbook/group_with_coder.py       |    6 +-
 .../examples/snippets/snippets_test.py          |    3 +-
 sdks/python/apache_beam/examples/wordcount.py   |   48 +-
 .../apache_beam/examples/wordcount_debugging.py |    4 -
 sdks/python/apache_beam/internal/gcp/auth.py    |   80 +-
 .../apache_beam/internal/gcp/auth_test.py       |   44 -
 .../apache_beam/internal/gcp/json_value.py      |    6 -
 sdks/python/apache_beam/internal/pickler.py     |   20 +-
 sdks/python/apache_beam/io/avroio_test.py       |   14 +-
 sdks/python/apache_beam/io/concat_source.py     |   74 +-
 .../python/apache_beam/io/concat_source_test.py |   12 +-
 sdks/python/apache_beam/io/filebasedsource.py   |   54 +-
 .../apache_beam/io/filebasedsource_test.py      |   26 +-
 sdks/python/apache_beam/io/fileio.py            |   72 +-
 sdks/python/apache_beam/io/fileio_test.py       |   45 +-
 sdks/python/apache_beam/io/filesystem.py        |    3 +-
 sdks/python/apache_beam/io/filesystem_test.py   |  242 ++-
 sdks/python/apache_beam/io/filesystems_util.py  |   10 +-
 sdks/python/apache_beam/io/gcp/bigquery.py      |   38 +-
 .../io/gcp/datastore/v1/datastoreio.py          |   24 +-
 .../io/gcp/datastore/v1/datastoreio_test.py     |    4 +-
 .../apache_beam/io/gcp/datastore/v1/helper.py   |   16 +-
 .../io/gcp/datastore/v1/query_splitter.py       |    2 +-
 sdks/python/apache_beam/io/gcp/gcsfilesystem.py |    7 +-
 .../io/gcp/tests/bigquery_matcher.py            |    3 +-
 sdks/python/apache_beam/io/iobase.py            |    7 +-
 sdks/python/apache_beam/io/localfilesystem.py   |    3 +-
 sdks/python/apache_beam/io/range_trackers.py    |   19 +-
 sdks/python/apache_beam/io/source_test_utils.py |   79 +-
 .../apache_beam/io/source_test_utils_test.py    |   20 +-
 sdks/python/apache_beam/io/textio.py            |   16 +-
 sdks/python/apache_beam/io/textio_test.py       |   18 +-
 sdks/python/apache_beam/io/tfrecordio.py        |   12 +-
 sdks/python/apache_beam/metrics/cells.py        |   28 +-
 sdks/python/apache_beam/metrics/execution.py    |    3 +-
 sdks/python/apache_beam/metrics/metric.py       |    9 +-
 sdks/python/apache_beam/pipeline.py             |    4 +-
 sdks/python/apache_beam/pipeline_test.py        |   13 +-
 sdks/python/apache_beam/pvalue.py               |   16 +-
 sdks/python/apache_beam/runners/common.py       |    9 +-
 .../runners/dataflow/dataflow_metrics_test.py   |    3 +-
 .../runners/dataflow/dataflow_runner.py         |   39 +-
 .../runners/dataflow/dataflow_runner_test.py    |    2 +-
 .../runners/dataflow/internal/apiclient.py      |    7 +-
 .../runners/dataflow/internal/dependency.py     |    6 +-
 .../runners/dataflow/native_io/iobase_test.py   |    2 +-
 .../runners/dataflow/test_dataflow_runner.py    |   14 +-
 .../runners/direct/bundle_factory.py            |   14 +-
 .../consumer_tracking_pipeline_visitor_test.py  |   22 +-
 .../apache_beam/runners/direct/direct_runner.py |   27 +-
 .../runners/direct/evaluation_context.py        |   10 +-
 .../apache_beam/runners/direct/executor.py      |    9 +-
 .../runners/direct/transform_evaluator.py       |   38 -
 sdks/python/apache_beam/runners/runner.py       |   80 +-
 sdks/python/apache_beam/runners/runner_test.py  |   41 +
 .../apache_beam/tests/pipeline_verifiers.py     |    7 +-
 sdks/python/apache_beam/transforms/combiners.py |   62 +-
 .../apache_beam/transforms/combiners_test.py    |    4 +-
 sdks/python/apache_beam/transforms/core.py      |  192 ++-
 .../apache_beam/transforms/create_test.py       |  121 ++
 sdks/python/apache_beam/transforms/display.py   |    4 +-
 .../apache_beam/transforms/display_test.py      |   36 -
 .../python/apache_beam/transforms/ptransform.py |   38 +-
 .../apache_beam/transforms/ptransform_test.py   |   17 +-
 .../python/apache_beam/transforms/sideinputs.py |   11 +-
 .../apache_beam/transforms/sideinputs_test.py   |    6 +-
 sdks/python/apache_beam/transforms/trigger.py   |   26 +-
 .../apache_beam/transforms/trigger_test.py      |    6 +-
 sdks/python/apache_beam/typehints/decorators.py |   26 +-
 .../apache_beam/typehints/trivial_inference.py  |   26 +-
 .../typehints/trivial_inference_test.py         |    3 +-
 sdks/python/apache_beam/typehints/typecheck.py  |    7 +-
 sdks/python/apache_beam/typehints/typehints.py  |   66 +-
 .../apache_beam/typehints/typehints_test.py     |    7 +-
 .../apache_beam/utils/annotations_test.py       |    2 +-
 sdks/python/apache_beam/utils/path.py           |    3 +-
 .../apache_beam/utils/pipeline_options.py       |  109 +-
 .../apache_beam/utils/pipeline_options_test.py  |   52 +-
 sdks/python/apache_beam/utils/proto_utils.py    |   15 +-
 sdks/python/apache_beam/utils/retry.py          |   14 +-
 sdks/python/apache_beam/utils/timestamp.py      |    6 +-
 sdks/python/apache_beam/utils/value_provider.py |  110 --
 .../apache_beam/utils/value_provider_test.py    |  165 --
 sdks/python/apache_beam/utils/windowed_value.py |   17 +-
 sdks/python/run_postcommit.sh                   |    6 +-
 sdks/python/run_pylint.sh                       |    2 +-
 sdks/python/setup.py                            |    8 +-
 sdks/python/tox.ini                             |   18 +-
 490 files changed, 17307 insertions(+), 17743 deletions(-)
----------------------------------------------------------------------



Mime
View raw message