beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [50/50] incubator-beam git commit: This closes #1416
Date Tue, 22 Nov 2016 20:11:54 GMT
This closes #1416


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b4187bd9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b4187bd9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b4187bd9

Branch: refs/heads/python-sdk
Commit: b4187bd91e9e53c1562ee845ffa87cc9e734006f
Parents: 28bfd90 0a66721
Author: Kenneth Knowles <klk@google.com>
Authored: Tue Nov 22 12:08:38 2016 -0800
Committer: Kenneth Knowles <klk@google.com>
Committed: Tue Nov 22 12:08:38 2016 -0800

----------------------------------------------------------------------
 .travis.yml                                     |  12 +-
 README.md                                       |   4 +-
 examples/java/pom.xml                           | 185 ++--
 .../beam/examples/DebuggingWordCount.java       |  69 +-
 .../apache/beam/examples/MinimalWordCount.java  |  52 +-
 .../org/apache/beam/examples/WordCount.java     |  64 +-
 .../beam/examples/common/ExampleUtils.java      |   3 +-
 .../apache/beam/examples/complete/TfIdf.java    |   8 +-
 .../examples/complete/TopWikipediaSessions.java |   2 +-
 .../examples/cookbook/BigQueryTornadoes.java    |   2 +-
 .../cookbook/CombinePerKeyExamples.java         |   2 +-
 .../beam/examples/cookbook/DeDupExample.java    |  10 +-
 .../beam/examples/cookbook/FilterExamples.java  |   2 +-
 .../beam/examples/cookbook/JoinExamples.java    |   2 +-
 .../examples/cookbook/MaxPerKeyExamples.java    |   2 +-
 .../org/apache/beam/examples/cookbook/README.md |   2 +-
 .../beam/examples/DebuggingWordCountTest.java   |  15 +-
 .../beam/examples/WindowedWordCountIT.java      |  31 +-
 .../org/apache/beam/examples/WordCountIT.java   |  21 +-
 .../org/apache/beam/examples/WordCountTest.java |   2 +-
 .../examples/complete/AutoCompleteTest.java     |   6 +-
 .../beam/examples/complete/TfIdfTest.java       |   6 +-
 .../complete/TopWikipediaSessionsTest.java      |   2 +-
 .../examples/cookbook/BigQueryTornadoesIT.java  |  15 +-
 .../examples/cookbook/DeDupExampleTest.java     |  82 --
 .../examples/cookbook/DistinctExampleTest.java  |  82 ++
 .../examples/cookbook/JoinExamplesTest.java     |   2 +-
 .../examples/cookbook/TriggerExampleTest.java   |   2 +-
 examples/java8/pom.xml                          |  32 +-
 .../beam/examples/MinimalWordCountJava8.java    |   2 +-
 .../beam/examples/complete/game/GameStats.java  |  11 +-
 .../examples/complete/game/HourlyTeamScore.java |  10 +-
 .../examples/complete/game/LeaderBoard.java     |  12 +-
 .../beam/examples/complete/game/UserScore.java  |  10 +-
 .../examples/MinimalWordCountJava8Test.java     |   2 +-
 .../examples/complete/game/GameStatsTest.java   |   7 +-
 .../complete/game/HourlyTeamScoreTest.java      |   7 +-
 .../examples/complete/game/LeaderBoardTest.java |  16 +-
 .../examples/complete/game/UserScoreTest.java   |   6 +-
 examples/pom.xml                                |  18 +-
 pom.xml                                         | 168 +++-
 runners/apex/README.md                          |  76 ++
 runners/apex/pom.xml                            | 234 ++++++
 .../beam/runners/apex/ApexPipelineOptions.java  |  60 ++
 .../apache/beam/runners/apex/ApexRunner.java    | 398 +++++++++
 .../beam/runners/apex/ApexRunnerRegistrar.java  |  61 ++
 .../beam/runners/apex/ApexRunnerResult.java     | 110 +++
 .../beam/runners/apex/TestApexRunner.java       |  73 ++
 .../apache/beam/runners/apex/package-info.java  |  22 +
 .../translation/ApexPipelineTranslator.java     | 179 ++++
 .../translation/CreateValuesTranslator.java     |  48 ++
 .../FlattenPCollectionTranslator.java           | 129 +++
 .../apex/translation/GroupByKeyTranslator.java  |  42 +
 .../translation/ParDoBoundMultiTranslator.java  | 142 ++++
 .../apex/translation/ParDoBoundTranslator.java  |  64 ++
 .../translation/ReadUnboundedTranslator.java    |  42 +
 .../apex/translation/TransformTranslator.java   |  31 +
 .../apex/translation/TranslationContext.java    | 178 ++++
 .../operators/ApexFlattenOperator.java          | 125 +++
 .../operators/ApexGroupByKeyOperator.java       | 489 +++++++++++
 .../operators/ApexParDoOperator.java            | 375 +++++++++
 .../ApexReadUnboundedInputOperator.java         | 155 ++++
 .../translation/operators/package-info.java     |  22 +
 .../runners/apex/translation/package-info.java  |  22 +
 .../translation/utils/ApexStateInternals.java   | 438 ++++++++++
 .../apex/translation/utils/ApexStreamTuple.java | 222 +++++
 .../utils/CoderAdapterStreamCodec.java          |  69 ++
 .../apex/translation/utils/NoOpStepContext.java |  72 ++
 .../utils/SerializablePipelineOptions.java      |  60 ++
 .../utils/ValueAndCoderKryoSerializable.java    |  77 ++
 .../apex/translation/utils/ValuesSource.java    | 149 ++++
 .../apex/translation/utils/package-info.java    |  22 +
 .../runners/apex/ApexRunnerRegistrarTest.java   |  47 ++
 .../apex/examples/UnboundedTextSource.java      | 142 ++++
 .../runners/apex/examples/WordCountTest.java    | 188 +++++
 .../runners/apex/examples/package-info.java     |  22 +
 .../translation/ApexGroupByKeyOperatorTest.java | 117 +++
 .../FlattenPCollectionTranslatorTest.java       |  99 +++
 .../translation/GroupByKeyTranslatorTest.java   | 246 ++++++
 .../translation/ParDoBoundTranslatorTest.java   | 340 ++++++++
 .../translation/ReadUnboundTranslatorTest.java  | 129 +++
 .../utils/ApexStateInternalsTest.java           | 361 ++++++++
 .../translation/utils/CollectionSource.java     | 136 +++
 .../translation/utils/PipelineOptionsTest.java  |  84 ++
 .../apex/src/test/resources/log4j.properties    |  35 +
 runners/apex/src/test/resources/words.txt       |   3 +
 runners/core-java/pom.xml                       |  67 +-
 .../core/GroupAlsoByWindowViaWindowSetDoFn.java |   6 +-
 .../GroupAlsoByWindowsViaOutputBufferDoFn.java  |   8 +-
 .../core/GroupByKeyViaGroupByKeyOnly.java       |  13 +-
 .../beam/runners/core/OutputWindowedValue.java  |  45 +
 .../beam/runners/core/PaneInfoTracker.java      |   3 -
 .../core/PushbackSideInputDoFnRunner.java       |  28 +-
 .../runners/core/ReduceFnContextFactory.java    |  83 +-
 .../beam/runners/core/ReduceFnRunner.java       |  40 +-
 .../beam/runners/core/SimpleDoFnRunner.java     | 158 ++--
 .../beam/runners/core/SimpleOldDoFnRunner.java  |  21 +-
 .../beam/runners/core/SplittableParDo.java      |  60 +-
 .../beam/runners/core/SystemReduceFn.java       |   3 -
 .../core/UnboundedReadFromBoundedSource.java    |   3 +-
 .../apache/beam/runners/core/WatermarkHold.java |   3 -
 .../core/WindowingInternalsAdapters.java        |  76 ++
 .../AfterDelayFromFirstElementStateMachine.java |   5 -
 .../core/triggers/AfterPaneStateMachine.java    |   3 -
 .../core/triggers/TriggerStateMachines.java     |   1 -
 .../beam/runners/core/ReduceFnTester.java       |  87 +-
 .../UnboundedReadFromBoundedSourceTest.java     |   4 +-
 runners/direct-java/pom.xml                     | 104 +--
 .../runners/direct/AggregatorContainer.java     |  22 +-
 .../direct/BoundedReadEvaluatorFactory.java     | 110 ++-
 ...ectGBKIntoKeyedWorkItemsOverrideFactory.java |  16 +-
 .../beam/runners/direct/DirectGroupByKey.java   |  42 +-
 .../direct/DirectGroupByKeyOverrideFactory.java |  25 +-
 .../beam/runners/direct/DirectRunner.java       | 159 ++--
 .../runners/direct/DirectTimerInternals.java    |  13 +
 .../runners/direct/DoFnLifecycleManager.java    |   4 +-
 .../beam/runners/direct/EmptyInputProvider.java |  22 +-
 .../direct/EncodabilityEnforcementFactory.java  |  80 --
 .../beam/runners/direct/EvaluationContext.java  |  18 +-
 .../direct/ExecutorServiceParallelExecutor.java | 107 +--
 .../GroupAlsoByWindowEvaluatorFactory.java      | 227 +++--
 .../direct/GroupByKeyOnlyEvaluatorFactory.java  |  31 +-
 .../ImmutabilityCheckingBundleFactory.java      |  11 +-
 .../direct/ImmutabilityEnforcementFactory.java  |   2 -
 .../direct/ImmutableListBundleFactory.java      |  21 +-
 .../direct/PTransformOverrideFactory.java       |   8 +-
 .../beam/runners/direct/ParDoEvaluator.java     |  28 +-
 .../runners/direct/ParDoEvaluatorFactory.java   | 108 +++
 .../direct/ParDoMultiEvaluatorFactory.java      | 107 ---
 .../direct/ParDoMultiOverrideFactory.java       |  51 ++
 .../runners/direct/ParDoOverrideFactory.java    |  55 --
 .../direct/ParDoSingleEvaluatorFactory.java     | 110 ---
 .../ParDoSingleViaMultiOverrideFactory.java     |  66 ++
 .../beam/runners/direct/RootInputProvider.java  |  16 +-
 .../runners/direct/RootProviderRegistry.java    |  19 +-
 .../runners/direct/StepTransformResult.java     |   6 +
 .../direct/TestStreamEvaluatorFactory.java      |  37 +-
 .../direct/TransformEvaluatorRegistry.java      |  10 +-
 .../beam/runners/direct/TransformExecutor.java  |  23 -
 .../direct/UnboundedReadEvaluatorFactory.java   |  38 +-
 .../direct/UncommittedBundleOutputManager.java  |  50 --
 .../runners/direct/ViewEvaluatorFactory.java    |  19 +-
 .../direct/WatermarkCallbackExecutor.java       |   7 +-
 .../beam/runners/direct/WatermarkManager.java   | 244 ++++--
 .../direct/WriteWithShardingFactory.java        |  15 +-
 .../direct/BoundedReadEvaluatorFactoryTest.java | 186 ++++-
 .../direct/CloningBundleFactoryTest.java        | 122 ++-
 .../ConsumerTrackingPipelineVisitorTest.java    |  17 +
 .../beam/runners/direct/DirectRunnerTest.java   |  88 ++
 .../EncodabilityEnforcementFactoryTest.java     | 323 -------
 .../runners/direct/EvaluationContextTest.java   |  23 +-
 .../direct/GroupByKeyEvaluatorFactoryTest.java  |  29 +-
 .../GroupByKeyOnlyEvaluatorFactoryTest.java     |  31 +-
 .../direct/ImmutableListBundleFactoryTest.java  |  15 +-
 .../beam/runners/direct/ParDoEvaluatorTest.java |  16 +-
 .../direct/ParDoMultiEvaluatorFactoryTest.java  | 439 ----------
 .../direct/ParDoSingleEvaluatorFactoryTest.java | 335 --------
 .../runners/direct/TransformExecutorTest.java   |  43 -
 .../UnboundedReadEvaluatorFactoryTest.java      |  47 +-
 .../runners/direct/WatermarkManagerTest.java    | 185 ++--
 .../direct/WriteWithShardingFactoryTest.java    |  26 +-
 runners/flink/examples/pom.xml                  |  17 -
 .../beam/runners/flink/examples/TFIDF.java      |  17 +-
 .../flink/examples/streaming/AutoComplete.java  |   8 +-
 .../flink/examples/streaming/JoinExamples.java  |  12 +-
 .../examples/streaming/KafkaIOExamples.java     |   2 +-
 .../examples/streaming/WindowedWordCount.java   |   8 +-
 runners/flink/pom.xml                           |   1 +
 runners/flink/runner/pom.xml                    |  17 -
 .../apache/beam/runners/flink/FlinkRunner.java  |   4 +-
 .../FlinkStreamingTransformTranslators.java     |  17 +-
 .../functions/FlinkDoFnFunction.java            |  21 +-
 .../FlinkMergingNonShuffleReduceFunction.java   |  14 +-
 .../FlinkMergingPartialReduceFunction.java      |  14 +-
 .../functions/FlinkMergingReduceFunction.java   |  12 +-
 .../functions/FlinkMultiOutputDoFnFunction.java |  14 +-
 .../FlinkMultiOutputProcessContext.java         |  94 +--
 .../functions/FlinkPartialReduceFunction.java   |  14 +-
 .../functions/FlinkProcessContext.java          | 330 --------
 .../functions/FlinkProcessContextBase.java      | 278 ++++++
 .../functions/FlinkReduceFunction.java          |  14 +-
 .../FlinkSingleOutputProcessContext.java        |  69 ++
 .../SerializableFnAggregatorWrapper.java        |   7 +
 .../wrappers/streaming/DoFnOperator.java        |  10 +-
 .../wrappers/streaming/WindowDoFnOperator.java  |  15 +-
 .../streaming/io/UnboundedSocketSource.java     |   5 +-
 .../streaming/io/UnboundedSourceWrapper.java    |   3 +-
 runners/google-cloud-dataflow-java/pom.xml      | 144 +---
 .../dataflow/BlockingDataflowRunner.java        | 170 ----
 .../dataflow/DataflowJobCancelledException.java |  39 -
 .../dataflow/DataflowJobExecutionException.java |  35 -
 .../dataflow/DataflowJobUpdatedException.java   |  51 --
 .../runners/dataflow/DataflowPipelineJob.java   |  44 +-
 .../dataflow/DataflowPipelineRegistrar.java     |  11 +-
 .../dataflow/DataflowPipelineTranslator.java    |   8 +-
 .../beam/runners/dataflow/DataflowRunner.java   |  51 +-
 .../runners/dataflow/internal/IsmFormat.java    |   5 +-
 .../dataflow/internal/ReadTranslator.java       |  12 +-
 .../BlockingDataflowPipelineOptions.java        |  28 -
 .../options/DataflowPipelineOptions.java        |   8 +
 .../testing/TestDataflowPipelineOptions.java    |   6 +-
 .../dataflow/testing/TestDataflowRunner.java    |  27 +-
 .../dataflow/util/DataflowTemplateJob.java      |  70 ++
 .../dataflow/util/DataflowTransport.java        |  13 +-
 .../beam/runners/dataflow/util/DoFnInfo.java    |  53 +-
 .../runners/dataflow/util/MonitoringUtil.java   |   1 -
 .../runners/dataflow/util/RandomAccessData.java |   2 +-
 .../dataflow/BlockingDataflowRunnerTest.java    | 300 -------
 .../dataflow/DataflowPipelineJobTest.java       |  30 +-
 .../dataflow/DataflowPipelineRegistrarTest.java |   7 +-
 .../DataflowPipelineTranslatorTest.java         |  28 +
 .../runners/dataflow/DataflowRunnerTest.java    |  45 +
 .../options/DataflowPipelineOptionsTest.java    |   6 +-
 .../runners/dataflow/util/PackageUtilTest.java  |   2 +-
 runners/pom.xml                                 |  17 +
 runners/spark/pom.xml                           |  24 +-
 .../beam/runners/spark/SparkContextOptions.java |  64 ++
 .../runners/spark/SparkPipelineOptions.java     |  49 +-
 .../apache/beam/runners/spark/SparkRunner.java  |   9 +-
 .../apache/beam/runners/spark/io/SourceRDD.java |   4 +-
 .../spark/translation/BoundedDataset.java       | 114 +++
 .../beam/runners/spark/translation/Dataset.java |  34 +
 .../runners/spark/translation/DoFnFunction.java |  10 +-
 .../spark/translation/EvaluationContext.java    | 257 +++---
 .../translation/GroupCombineFunctions.java      |   5 +-
 .../spark/translation/MultiDoFnFunction.java    |  21 +-
 .../ReifyTimestampsAndWindowsFunction.java      |  47 ++
 .../spark/translation/SparkContextFactory.java  |  19 +-
 .../spark/translation/SparkProcessContext.java  | 125 +--
 .../spark/translation/SparkRuntimeContext.java  |   4 +-
 .../spark/translation/TransformTranslator.java  |  99 +--
 .../SparkRunnerStreamingContextFactory.java     |  34 +-
 .../streaming/StreamingEvaluationContext.java   | 272 ------
 .../streaming/StreamingTransformTranslator.java | 135 ++-
 .../translation/streaming/UnboundedDataset.java | 103 +++
 .../runners/spark/ProvidedSparkContextTest.java |  24 +-
 .../metrics/sink/NamedAggregatorsTest.java      |  13 +-
 .../beam/runners/spark/io/AvroPipelineTest.java |  11 +-
 .../beam/runners/spark/io/NumShardsTest.java    |  11 +-
 .../io/hadoop/HadoopFileFormatPipelineTest.java |  11 +-
 .../spark/translation/SideEffectsTest.java      |  41 +-
 .../streaming/EmptyStreamAssertionTest.java     |  11 +-
 .../streaming/FlattenStreamingTest.java         |  15 +-
 .../streaming/KafkaStreamingTest.java           |  28 +-
 .../ResumeFromCheckpointStreamingTest.java      |  11 +-
 .../streaming/SimpleStreamingWordCountTest.java |  12 +-
 .../streaming/utils/PAssertStreaming.java       |   8 +-
 .../utils/SparkTestPipelineOptions.java         |  42 +
 .../SparkTestPipelineOptionsForStreaming.java   |  43 +
 .../utils/TestOptionsForStreaming.java          |  55 --
 sdks/java/build-tools/pom.xml                   |   4 +-
 .../src/main/resources/beam/checkstyle.xml      |   9 +-
 .../src/main/resources/beam/findbugs-filter.xml | 172 +++-
 sdks/java/core/pom.xml                          | 166 +---
 .../main/java/org/apache/beam/sdk/Pipeline.java |  17 -
 .../org/apache/beam/sdk/coders/AvroCoder.java   | 121 ++-
 .../apache/beam/sdk/coders/InstantCoder.java    |   4 -
 .../java/org/apache/beam/sdk/io/AvroIO.java     |   4 +-
 .../sdk/io/BoundedReadFromUnboundedSource.java  |   4 +-
 .../org/apache/beam/sdk/io/BoundedSource.java   |  11 +-
 .../apache/beam/sdk/io/CompressedSource.java    |   4 +-
 .../org/apache/beam/sdk/io/FileBasedSink.java   | 295 ++-----
 .../org/apache/beam/sdk/io/FileBasedSource.java |  59 +-
 .../beam/sdk/io/PubsubUnboundedSource.java      | 123 ++-
 .../main/java/org/apache/beam/sdk/io/Sink.java  |   4 +-
 .../java/org/apache/beam/sdk/io/TextIO.java     |  43 +-
 .../java/org/apache/beam/sdk/io/XmlSource.java  |   2 +-
 .../apache/beam/sdk/io/range/ByteKeyRange.java  |   2 +-
 .../beam/sdk/metrics/MetricsContainer.java      |  13 -
 .../org/apache/beam/sdk/options/GcpOptions.java | 136 +--
 .../sdk/options/PipelineOptionsFactory.java     | 280 +++++--
 .../sdk/options/ProxyInvocationHandler.java     |   5 +-
 .../apache/beam/sdk/options/ValueProvider.java  |  62 +-
 .../apache/beam/sdk/runners/PipelineRunner.java |   2 +-
 .../beam/sdk/testing/BigqueryMatcher.java       |  15 +-
 .../org/apache/beam/sdk/testing/PAssert.java    |   3 -
 .../beam/sdk/testing/SerializableMatchers.java  |   3 -
 .../apache/beam/sdk/testing/TestPipeline.java   |   2 +-
 .../beam/sdk/testing/TestPipelineOptions.java   |   6 +
 .../apache/beam/sdk/transforms/Aggregator.java  |  13 +-
 .../apache/beam/sdk/transforms/CombineFns.java  |   4 +-
 .../sdk/transforms/DelegatingAggregator.java    |   7 +-
 .../apache/beam/sdk/transforms/Distinct.java    | 158 ++++
 .../org/apache/beam/sdk/transforms/DoFn.java    | 111 +--
 .../beam/sdk/transforms/DoFnAdapters.java       |  70 +-
 .../apache/beam/sdk/transforms/DoFnTester.java  |  18 +-
 .../org/apache/beam/sdk/transforms/Flatten.java |   3 +-
 .../apache/beam/sdk/transforms/GroupByKey.java  |   4 +-
 .../org/apache/beam/sdk/transforms/OldDoFn.java |   7 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  51 +-
 .../org/apache/beam/sdk/transforms/Regex.java   | 454 ++++++++++
 .../beam/sdk/transforms/RemoveDuplicates.java   | 159 ----
 .../org/apache/beam/sdk/transforms/View.java    |   2 +-
 .../beam/sdk/transforms/WithTimestamps.java     |   5 +-
 .../sdk/transforms/display/DisplayData.java     |   5 +-
 .../beam/sdk/transforms/join/CoGroupByKey.java  |   6 +-
 .../reflect/ByteBuddyDoFnInvokerFactory.java    | 837 +++++++++++++++++++
 .../reflect/ByteBuddyOnTimerInvokerFactory.java | 307 +++++++
 .../sdk/transforms/reflect/DoFnInvoker.java     | 123 ++-
 .../transforms/reflect/DoFnInvokerFactory.java  |  27 +
 .../sdk/transforms/reflect/DoFnInvokers.java    | 673 +--------------
 .../sdk/transforms/reflect/DoFnSignature.java   | 163 +++-
 .../sdk/transforms/reflect/DoFnSignatures.java  | 826 +++++++++++-------
 .../sdk/transforms/reflect/OnTimerInvoker.java  |  27 +
 .../reflect/OnTimerInvokerFactory.java          |  36 +
 .../sdk/transforms/reflect/OnTimerInvokers.java |  42 +
 .../splittabledofn/RestrictionTracker.java      |   2 +-
 .../transforms/splittabledofn/package-info.java |   4 +-
 .../beam/sdk/transforms/windowing/Window.java   | 130 +--
 .../beam/sdk/transforms/windowing/WindowFn.java |  12 +
 ...AttemptAndTimeBoundedExponentialBackOff.java |   3 -
 .../beam/sdk/util/BaseExecutionContext.java     |   2 +-
 .../BufferedElementCountingOutputStream.java    |   4 +-
 .../apache/beam/sdk/util/CredentialFactory.java |   4 +-
 .../org/apache/beam/sdk/util/Credentials.java   | 192 -----
 .../apache/beam/sdk/util/ExecutableTrigger.java | 131 ---
 .../apache/beam/sdk/util/ExecutionContext.java  |   8 +-
 .../sdk/util/ExposedByteArrayInputStream.java   |   7 +-
 .../sdk/util/ExposedByteArrayOutputStream.java  |   5 -
 .../beam/sdk/util/FileIOChannelFactory.java     |  74 +-
 .../sdk/util/FileIOChannelFactoryRegistrar.java |  38 +
 .../apache/beam/sdk/util/GatherAllPanes.java    |  18 +-
 .../beam/sdk/util/GcpCredentialFactory.java     |  42 +-
 .../beam/sdk/util/GcsIOChannelFactory.java      |  28 +-
 .../sdk/util/GcsIOChannelFactoryRegistrar.java  |  38 +
 .../java/org/apache/beam/sdk/util/GcsUtil.java  |   2 -
 .../apache/beam/sdk/util/IOChannelFactory.java  |  26 +
 .../sdk/util/IOChannelFactoryRegistrar.java     |  48 ++
 .../apache/beam/sdk/util/IOChannelUtils.java    | 134 ++-
 .../apache/beam/sdk/util/MovingFunction.java    |   4 +-
 .../beam/sdk/util/NoopCredentialFactory.java    |  40 +-
 .../sdk/util/NullCredentialInitializer.java     |  62 ++
 .../org/apache/beam/sdk/util/PubsubClient.java  |  32 +-
 .../apache/beam/sdk/util/PubsubGrpcClient.java  |  18 +-
 .../apache/beam/sdk/util/PubsubJsonClient.java  |   9 +-
 .../apache/beam/sdk/util/PubsubTestClient.java  |  32 +
 .../sdk/util/ReifyTimestampAndWindowsDoFn.java  |  41 -
 .../sdk/util/ReifyTimestampsAndWindows.java     |  63 --
 .../org/apache/beam/sdk/util/StringUtils.java   |   8 +-
 .../apache/beam/sdk/util/TestCredential.java    |  44 +-
 .../apache/beam/sdk/util/TimerInternals.java    |  48 +-
 .../org/apache/beam/sdk/util/Transport.java     |  18 +-
 .../beam/sdk/util/WindowingInternals.java       |  14 +-
 .../apache/beam/sdk/util/WindowingStrategy.java |  10 +-
 .../java/org/apache/beam/sdk/util/ZipFiles.java |   9 -
 .../beam/sdk/util/common/ReflectHelpers.java    |  46 +
 .../org/apache/beam/sdk/util/gcsfs/GcsPath.java |  13 +-
 .../sdk/util/state/InMemoryTimerInternals.java  |  12 +
 .../beam/sdk/util/state/StateContexts.java      |  20 +-
 .../beam/sdk/util/state/StateMerging.java       |   3 -
 .../beam/sdk/util/state/StateNamespace.java     |  16 +-
 .../java/org/apache/beam/sdk/values/PDone.java  |   2 +-
 .../apache/beam/sdk/values/TypeDescriptor.java  |  48 +-
 .../apache/beam/sdk/values/TypeDescriptors.java |   1 -
 .../apache/beam/sdk/values/TypeParameter.java   |  62 ++
 .../apache/beam/sdk/coders/AvroCoderTest.java   |  48 ++
 .../java/org/apache/beam/sdk/io/AvroIOTest.java |   2 +-
 .../apache/beam/sdk/io/CountingInputTest.java   |   6 +-
 .../apache/beam/sdk/io/CountingSourceTest.java  |   8 +-
 .../apache/beam/sdk/io/FileBasedSinkTest.java   | 102 +--
 .../apache/beam/sdk/io/FileBasedSourceTest.java |   2 +-
 .../beam/sdk/io/PubsubUnboundedSourceTest.java  |  79 ++
 .../java/org/apache/beam/sdk/io/TextIOTest.java |   2 +-
 .../org/apache/beam/sdk/io/XmlSinkTest.java     |  16 +-
 .../sdk/options/PipelineOptionsFactoryTest.java | 273 +++++-
 .../sdk/options/ProxyInvocationHandlerTest.java |  29 +-
 .../beam/sdk/options/ValueProviderTest.java     |  40 +
 .../apache/beam/sdk/testing/ExpectedLogs.java   |  15 +-
 .../beam/sdk/testing/ExpectedLogsTest.java      |  30 +-
 .../apache/beam/sdk/testing/PAssertTest.java    |  27 +
 .../sdk/transforms/ApproximateUniqueTest.java   |   2 +-
 .../beam/sdk/transforms/DistinctTest.java       | 130 +++
 .../apache/beam/sdk/transforms/FlattenTest.java |  38 +-
 .../apache/beam/sdk/transforms/OldDoFnTest.java |  11 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   | 116 ++-
 .../apache/beam/sdk/transforms/RegexTest.java   | 248 ++++++
 .../sdk/transforms/RemoveDuplicatesTest.java    | 130 ---
 .../transforms/reflect/DoFnInvokersTest.java    | 328 ++++++--
 .../DoFnSignaturesProcessElementTest.java       |  52 +-
 .../DoFnSignaturesSplittableDoFnTest.java       |  60 +-
 .../transforms/reflect/DoFnSignaturesTest.java  | 132 +--
 .../reflect/DoFnSignaturesTestUtils.java        |  13 +-
 .../transforms/reflect/OnTimerInvokersTest.java | 108 +++
 .../testhelper/DoFnInvokersTestHelper.java      | 137 +++
 .../sdk/transforms/windowing/WindowTest.java    |   6 +-
 .../beam/sdk/util/ExecutableTriggerTest.java    | 109 ---
 .../util/FileIOChannelFactoryRegistrarTest.java |  44 +
 .../beam/sdk/util/FileIOChannelFactoryTest.java |  10 +-
 .../util/GcsIOChannelFactoryRegistrarTest.java  |  44 +
 .../beam/sdk/util/GcsIOChannelFactoryTest.java  |   2 +-
 .../beam/sdk/util/IOChannelUtilsTest.java       |  39 +
 .../beam/sdk/util/PubsubGrpcClientTest.java     |   9 +-
 .../sdk/util/common/ReflectHelpersTest.java     |  58 ++
 .../apache/beam/sdk/util/gcsfs/GcsPathTest.java |  25 +
 .../beam/sdk/values/TypeDescriptorTest.java     |  12 +
 sdks/java/extensions/join-library/pom.xml       |   8 -
 sdks/java/extensions/pom.xml                    |   1 +
 sdks/java/extensions/sorter/README.md           |  42 +
 sdks/java/extensions/sorter/pom.xml             | 159 ++++
 .../sorter/BufferedExternalSorter.java          | 125 +++
 .../sdk/extensions/sorter/ExternalSorter.java   | 225 +++++
 .../sdk/extensions/sorter/InMemorySorter.java   | 166 ++++
 .../beam/sdk/extensions/sorter/SortValues.java  | 213 +++++
 .../beam/sdk/extensions/sorter/Sorter.java      |  47 ++
 .../sdk/extensions/sorter/package-info.java     |  23 +
 .../sorter/BufferedExternalSorterTest.java      | 177 ++++
 .../extensions/sorter/ExternalSorterTest.java   |  87 ++
 .../extensions/sorter/InMemorySorterTest.java   | 144 ++++
 .../sdk/extensions/sorter/SortValuesTest.java   | 128 +++
 .../sdk/extensions/sorter/SorterTestUtils.java  | 129 +++
 sdks/java/io/google-cloud-platform/pom.xml      |  31 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    |  10 +-
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java    |  98 ++-
 .../sdk/io/gcp/bigtable/BigtableService.java    |   6 +
 .../io/gcp/bigtable/BigtableServiceImpl.java    |   5 +
 .../beam/sdk/io/gcp/datastore/DatastoreV1.java  |  27 +-
 .../apache/beam/sdk/io/gcp/ApiSurfaceTest.java  | 132 +++
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     |   6 +-
 .../sdk/io/gcp/bigtable/BigtableIOTest.java     |  83 +-
 .../sdk/io/gcp/bigtable/BigtableWriteIT.java    |  26 +-
 .../beam/sdk/io/gcp/datastore/V1ReadIT.java     |  15 +-
 .../sdk/io/gcp/datastore/V1TestOptions.java     |   5 -
 .../beam/sdk/io/gcp/datastore/V1TestUtil.java   |  35 +-
 .../beam/sdk/io/gcp/datastore/V1WriteIT.java    |   9 +-
 sdks/java/io/hdfs/pom.xml                       |  10 +-
 sdks/java/io/jdbc/pom.xml                       |  16 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java     |  39 +
 .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java |  58 +-
 sdks/java/io/jms/pom.xml                        |  23 +-
 sdks/java/io/kafka/pom.xml                      |  30 +-
 .../beam/sdk/io/kafka/KafkaCheckpointMark.java  |  32 +-
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java   |  18 +-
 .../apache/beam/sdk/io/kafka/KafkaIOTest.java   |   6 +-
 sdks/java/io/kinesis/pom.xml                    |  25 +-
 .../beam/sdk/io/kinesis/CustomOptional.java     |  21 +-
 .../apache/beam/sdk/io/kinesis/KinesisIO.java   |  35 +-
 .../beam/sdk/io/kinesis/KinesisRecord.java      |   3 +-
 .../beam/sdk/io/kinesis/CustomOptionalTest.java |  10 +
 sdks/java/io/mongodb/pom.xml                    |  14 +-
 .../beam/sdk/io/mongodb/MongoDbGridFSIO.java    | 308 ++++++-
 .../apache/beam/sdk/io/mongodb/MongoDbIO.java   |   8 +-
 .../sdk/io/mongodb/MongoDBGridFSIOTest.java     | 121 ++-
 .../beam/sdk/io/mongodb/MongoDbIOTest.java      |  38 +-
 sdks/java/java8tests/pom.xml                    |  11 -
 .../beam/sdk/transforms/DistinctJava8Test.java  | 104 +++
 .../transforms/RemoveDuplicatesJava8Test.java   | 104 ---
 sdks/java/maven-archetypes/examples/pom.xml     |  42 +
 .../main/resources/archetype-resources/pom.xml  |  46 +-
 .../src/main/java/DebuggingWordCount.java       |  69 +-
 .../src/main/java/MinimalWordCount.java         |  52 +-
 .../src/main/java/WindowedWordCount.java        |   6 +-
 .../src/main/java/WordCount.java                |  64 +-
 .../common/ExampleBigQueryTableOptions.java     |   2 +-
 .../src/main/java/common/ExampleOptions.java    |   5 +
 ...xamplePubsubTopicAndSubscriptionOptions.java |   2 +-
 .../java/common/ExamplePubsubTopicOptions.java  |   2 +-
 .../src/main/java/common/ExampleUtils.java      |   3 +-
 .../src/test/java/DebuggingWordCountTest.java   |  15 +-
 .../src/test/java/WordCountTest.java            |   7 +-
 sdks/java/maven-archetypes/starter/pom.xml      |   6 +
 .../main/resources/archetype-resources/pom.xml  |  14 +
 .../resources/projects/basic/reference/pom.xml  |  14 +
 .../update-examples-archetype.sh                |  59 ++
 sdks/java/microbenchmarks/README.md             |  42 -
 sdks/java/microbenchmarks/pom.xml               | 110 ---
 .../coders/AvroCoderBenchmark.java              | 119 ---
 .../coders/ByteArrayCoderBenchmark.java         |  64 --
 .../coders/CoderBenchmarking.java               |  41 -
 .../coders/StringUtf8CoderBenchmark.java        |  70 --
 .../microbenchmarks/coders/package-info.java    |  22 -
 .../transforms/DoFnInvokersBenchmark.java       | 231 -----
 .../transforms/package-info.java                |  22 -
 sdks/java/pom.xml                               |  10 +-
 sdks/pom.xml                                    |  19 +
 474 files changed, 20040 insertions(+), 9983 deletions(-)
----------------------------------------------------------------------



Mime
View raw message