beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From taki...@apache.org
Subject [50/50] [abbrv] beam git commit: [BEAM-2610] This closes #3553
Date Thu, 13 Jul 2017 03:07:02 GMT
[BEAM-2610] This closes #3553


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ec494f67
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ec494f67
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ec494f67

Branch: refs/heads/DSL_SQL
Commit: ec494f675aa73fbdc7929f9592f33951941962b0
Parents: d89d1ee 4f7f169
Author: Tyler Akidau <takidau@apache.org>
Authored: Wed Jul 12 20:02:22 2017 -0700
Committer: Tyler Akidau <takidau@apache.org>
Committed: Wed Jul 12 20:02:22 2017 -0700

----------------------------------------------------------------------
 .gitignore                                      |   2 +-
 .../jenkins/common_job_properties.groovy        |   9 +-
 .../job_beam_PerformanceTests_Python.groovy     |  58 ++
 ..._beam_PostCommit_Java_JDKVersionsTest.groovy |   2 +
 ..._PostCommit_Java_MavenInstall_Windows.groovy |   3 +-
 .../job_beam_PreCommit_Website_Merge.groovy     |  59 ++
 examples/java/pom.xml                           |  20 +-
 .../examples/common/WriteOneFilePerWindow.java  |  52 +-
 .../beam/examples/WindowedWordCountIT.java      |   4 +-
 examples/java8/pom.xml                          |  20 +-
 .../complete/game/utils/WriteToText.java        |  43 +-
 .../examples/complete/game/LeaderBoardTest.java |   2 +
 examples/pom.xml                                |   2 +-
 pom.xml                                         | 123 +++-
 runners/apex/pom.xml                            |  20 +-
 .../apache/beam/runners/apex/ApexRunner.java    |  61 +-
 .../translation/ApexPipelineTranslator.java     |  16 +-
 .../apex/translation/TranslationContext.java    |   4 +-
 .../operators/ApexParDoOperator.java            |  21 +-
 .../runners/apex/examples/WordCountTest.java    |   8 +-
 .../utils/ApexStateInternalsTest.java           | 411 +++--------
 runners/core-construction-java/pom.xml          |   2 +-
 .../CreatePCollectionViewTranslation.java       |   4 +-
 .../construction/ElementAndRestriction.java     |  42 --
 .../ElementAndRestrictionCoder.java             |  88 ---
 .../construction/PCollectionTranslation.java    |  16 +
 .../core/construction/PTransformMatchers.java   | 109 ++-
 .../construction/PTransformTranslation.java     |   7 +-
 .../core/construction/ParDoTranslation.java     |  82 ++-
 .../construction/RunnerPCollectionView.java     |  31 +-
 .../core/construction/SplittableParDo.java      | 124 +++-
 .../construction/TestStreamTranslation.java     |  49 +-
 .../core/construction/TransformInputs.java      |  50 ++
 .../WindowingStrategyTranslation.java           |  27 +-
 .../construction/WriteFilesTranslation.java     |  67 +-
 .../ElementAndRestrictionCoderTest.java         | 126 ----
 .../PCollectionTranslationTest.java             |  22 +
 .../construction/PTransformMatchersTest.java    |  54 +-
 .../core/construction/ParDoTranslationTest.java |  28 +-
 .../core/construction/SplittableParDoTest.java  |  18 +-
 .../core/construction/TransformInputsTest.java  | 166 +++++
 .../WindowingStrategyTranslationTest.java       |   3 +
 .../construction/WriteFilesTranslationTest.java |  62 +-
 runners/core-java/pom.xml                       |   2 +-
 .../core/LateDataDroppingDoFnRunner.java        |  33 +-
 ...eBoundedSplittableProcessElementInvoker.java |  40 +-
 .../beam/runners/core/ProcessFnRunner.java      |  16 +-
 .../beam/runners/core/ReduceFnRunner.java       | 135 ++--
 .../beam/runners/core/SimpleDoFnRunner.java     |  20 +
 .../core/SplittableParDoViaKeyedWorkItems.java  |  58 +-
 .../core/SplittableProcessElementInvoker.java   |  25 +-
 .../beam/runners/core/SystemReduceFn.java       |   6 +
 .../core/triggers/AfterAllStateMachine.java     |  25 +-
 .../AfterDelayFromFirstElementStateMachine.java |   6 +-
 .../core/triggers/AfterFirstStateMachine.java   |  20 +-
 .../core/triggers/AfterPaneStateMachine.java    |   6 +-
 .../triggers/AfterWatermarkStateMachine.java    |   7 +-
 .../triggers/ExecutableTriggerStateMachine.java |  23 +-
 .../core/triggers/NeverStateMachine.java        |   5 +-
 .../core/triggers/TriggerStateMachine.java      |  27 -
 .../core/InMemoryStateInternalsTest.java        | 569 ++-------------
 ...ndedSplittableProcessElementInvokerTest.java |  47 +-
 .../beam/runners/core/ReduceFnRunnerTest.java   | 374 +++++++++-
 .../beam/runners/core/ReduceFnTester.java       |  48 +-
 .../core/SplittableParDoProcessFnTest.java      | 117 ++-
 .../beam/runners/core/StateInternalsTest.java   | 613 ++++++++++++++++
 .../beam/runners/core/WindowMatchers.java       |  15 +
 .../triggers/AfterFirstStateMachineTest.java    |   5 +-
 .../AfterWatermarkStateMachineTest.java         |   7 +-
 .../core/triggers/StubTriggerStateMachine.java  |   7 +-
 runners/direct-java/pom.xml                     |   7 +-
 .../beam/runners/direct/CommittedResult.java    |  12 +-
 .../apache/beam/runners/direct/DirectGraph.java |  38 +-
 .../beam/runners/direct/DirectGraphVisitor.java |  48 +-
 .../beam/runners/direct/DirectGroupByKey.java   |  13 +-
 .../direct/DirectGroupByKeyOverrideFactory.java |  14 +-
 .../beam/runners/direct/DirectRegistrar.java    |   2 +-
 .../beam/runners/direct/DirectRunner.java       |  64 +-
 .../beam/runners/direct/DirectTestOptions.java  |  42 ++
 .../beam/runners/direct/EvaluationContext.java  |  26 +-
 .../direct/ExecutorServiceParallelExecutor.java |  27 +-
 .../runners/direct/ParDoEvaluatorFactory.java   |   9 +-
 .../direct/ParDoMultiOverrideFactory.java       | 121 +++-
 ...littableProcessElementsEvaluatorFactory.java |  37 +-
 .../direct/StatefulParDoEvaluatorFactory.java   |  12 +-
 .../direct/TestStreamEvaluatorFactory.java      |  20 +-
 .../runners/direct/ViewEvaluatorFactory.java    |   8 +-
 .../runners/direct/ViewOverrideFactory.java     |  69 +-
 .../beam/runners/direct/WatermarkManager.java   |  18 +-
 .../direct/WriteWithShardingFactory.java        |  34 +-
 .../runners/direct/CommittedResultTest.java     |  17 +-
 .../runners/direct/DirectGraphVisitorTest.java  |  10 +-
 .../beam/runners/direct/DirectGraphs.java       |   7 +
 .../runners/direct/DirectRegistrarTest.java     |   2 +-
 .../runners/direct/EvaluationContextTest.java   |   7 +-
 .../ImmutabilityEnforcementFactoryTest.java     |   4 +-
 .../beam/runners/direct/ParDoEvaluatorTest.java |   7 +-
 .../StatefulParDoEvaluatorFactoryTest.java      |  65 +-
 .../runners/direct/TransformExecutorTest.java   |  12 +-
 .../direct/ViewEvaluatorFactoryTest.java        |   8 +-
 .../runners/direct/ViewOverrideFactoryTest.java |  37 +-
 .../direct/WatermarkCallbackExecutorTest.java   |   1 +
 .../runners/direct/WatermarkManagerTest.java    |  16 +-
 .../direct/WriteWithShardingFactoryTest.java    |  44 +-
 runners/flink/pom.xml                           |  11 +-
 .../runners/flink/CreateStreamingFlinkView.java | 154 ++++
 .../flink/FlinkBatchTranslationContext.java     |   3 +-
 .../FlinkPipelineExecutionEnvironment.java      |   2 +
 .../flink/FlinkStreamingPipelineTranslator.java |  86 +--
 .../FlinkStreamingTransformTranslators.java     |  36 +-
 .../flink/FlinkStreamingTranslationContext.java |   3 +-
 .../flink/FlinkStreamingViewOverrides.java      | 372 ----------
 .../runners/flink/FlinkTransformOverrides.java  |  53 ++
 .../streaming/SplittableDoFnOperator.java       |  16 +-
 .../streaming/state/FlinkStateInternals.java    | 425 +++++------
 .../FlinkBroadcastStateInternalsTest.java       | 242 ++-----
 .../FlinkKeyGroupStateInternalsTest.java        | 359 +++++-----
 .../streaming/FlinkSplitStateInternalsTest.java | 132 ++--
 .../streaming/FlinkStateInternalsTest.java      | 343 +--------
 runners/google-cloud-dataflow-java/pom.xml      |  10 +-
 .../dataflow/BatchStatefulParDoOverrides.java   |   4 +
 .../runners/dataflow/BatchViewOverrides.java    | 182 ++---
 .../runners/dataflow/CreateDataflowView.java    |   8 +-
 .../dataflow/DataflowPipelineTranslator.java    |  62 +-
 .../beam/runners/dataflow/DataflowRunner.java   | 133 +++-
 .../dataflow/SplittableParDoOverrides.java      |  76 ++
 .../dataflow/StreamingViewOverrides.java        |  10 +-
 .../runners/dataflow/TransformTranslator.java   |   4 +-
 .../runners/dataflow/util/PropertyNames.java    |   1 +
 .../beam/runners/dataflow/util/TimeUtil.java    |  24 +-
 .../DataflowPipelineTranslatorTest.java         |  95 ++-
 .../runners/dataflow/DataflowRunnerTest.java    | 198 +++++-
 .../runners/dataflow/util/TimeUtilTest.java     |   6 +
 runners/pom.xml                                 |   2 +-
 runners/spark/pom.xml                           |  70 +-
 .../spark/SparkNativePipelineVisitor.java       |   3 +-
 .../apache/beam/runners/spark/SparkRunner.java  |   9 +-
 .../beam/runners/spark/TestSparkRunner.java     |   2 +-
 .../SparkGroupAlsoByWindowViaWindowSet.java     |   6 +-
 .../spark/stateful/SparkTimerInternals.java     |  18 +-
 .../spark/translation/EvaluationContext.java    |   4 +-
 .../spark/translation/TransformTranslator.java  |  50 +-
 .../spark/util/GlobalWatermarkHolder.java       | 127 +++-
 .../spark/GlobalWatermarkHolderTest.java        |  18 +-
 .../runners/spark/SparkRunnerDebuggerTest.java  |  26 +-
 .../spark/stateful/SparkStateInternalsTest.java |  66 ++
 .../spark/translation/StorageLevelTest.java     |   4 +-
 sdks/common/fn-api/pom.xml                      |   2 +-
 .../fn-api/src/main/proto/beam_fn_api.proto     | 237 ++----
 sdks/common/pom.xml                             |   2 +-
 sdks/common/runner-api/pom.xml                  |   2 +-
 .../src/main/proto/beam_runner_api.proto        |  26 +-
 sdks/java/build-tools/pom.xml                   |   2 +-
 .../src/main/resources/beam/findbugs-filter.xml |   9 +
 sdks/java/core/pom.xml                          |   2 +-
 .../apache/beam/sdk/coders/ShardedKeyCoder.java |  66 ++
 .../java/org/apache/beam/sdk/io/AvroIO.java     | 220 +++---
 .../java/org/apache/beam/sdk/io/AvroSink.java   |  32 +-
 .../apache/beam/sdk/io/CompressedSource.java    |  40 +-
 .../beam/sdk/io/DefaultFilenamePolicy.java      | 274 +++++--
 .../beam/sdk/io/DynamicFileDestinations.java    | 115 +++
 .../org/apache/beam/sdk/io/FileBasedSink.java   | 513 +++++++------
 .../apache/beam/sdk/io/OffsetBasedSource.java   |  22 +-
 .../java/org/apache/beam/sdk/io/TFRecordIO.java |  44 +-
 .../java/org/apache/beam/sdk/io/TextIO.java     | 712 +++++++++++++++----
 .../java/org/apache/beam/sdk/io/TextSink.java   |  22 +-
 .../java/org/apache/beam/sdk/io/WriteFiles.java | 647 +++++++++++------
 .../beam/sdk/io/range/ByteKeyRangeTracker.java  |  22 +-
 .../apache/beam/sdk/io/range/OffsetRange.java   | 101 +++
 .../beam/sdk/io/range/OffsetRangeTracker.java   |   3 +
 .../sdk/options/PipelineOptionsFactory.java     |  18 +-
 .../sdk/options/PipelineOptionsValidator.java   |  34 +-
 .../sdk/options/ProxyInvocationHandler.java     |  19 +-
 .../beam/sdk/runners/TransformHierarchy.java    | 165 ++++-
 .../apache/beam/sdk/testing/StaticWindows.java  |   5 +
 .../org/apache/beam/sdk/testing/TestStream.java |  12 +
 .../org/apache/beam/sdk/transforms/Combine.java |  30 +-
 .../org/apache/beam/sdk/transforms/DoFn.java    |  52 +-
 .../apache/beam/sdk/transforms/DoFnTester.java  |  21 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  41 +-
 .../sdk/transforms/SerializableFunctions.java   |  50 ++
 .../org/apache/beam/sdk/transforms/View.java    |  38 +-
 .../reflect/ByteBuddyDoFnInvokerFactory.java    |  27 +
 .../reflect/ByteBuddyOnTimerInvokerFactory.java |  73 +-
 .../sdk/transforms/reflect/DoFnInvoker.java     |  17 +-
 .../sdk/transforms/reflect/DoFnSignature.java   |  33 +-
 .../sdk/transforms/reflect/DoFnSignatures.java  |  44 +-
 .../reflect/OnTimerMethodSpecifier.java         |  37 +
 .../transforms/splittabledofn/OffsetRange.java  |  77 --
 .../splittabledofn/OffsetRangeTracker.java      |  11 +
 .../splittabledofn/RestrictionTracker.java      |  11 +-
 .../sdk/transforms/windowing/GlobalWindows.java |   5 +
 .../windowing/PartitioningWindowFn.java         |   5 +
 .../transforms/windowing/SlidingWindows.java    |   5 +
 .../beam/sdk/transforms/windowing/Window.java   |  32 +
 .../beam/sdk/transforms/windowing/WindowFn.java |  11 +
 .../apache/beam/sdk/util/IdentityWindowFn.java  |   5 +
 .../org/apache/beam/sdk/values/PCollection.java |  12 +
 .../beam/sdk/values/PCollectionViews.java       |  38 +
 .../org/apache/beam/sdk/values/PValueBase.java  |  12 -
 .../org/apache/beam/sdk/values/ShardedKey.java  |  65 ++
 .../beam/sdk/values/WindowingStrategy.java      |  46 +-
 .../java/org/apache/beam/sdk/io/AvroIOTest.java |  85 ++-
 .../beam/sdk/io/DefaultFilenamePolicyTest.java  | 135 ++--
 .../sdk/io/DrunkWritableByteChannelFactory.java |   2 +-
 .../apache/beam/sdk/io/FileBasedSinkTest.java   |  93 ++-
 .../java/org/apache/beam/sdk/io/SimpleSink.java |  56 +-
 .../java/org/apache/beam/sdk/io/TextIOTest.java | 326 ++++++++-
 .../org/apache/beam/sdk/io/WriteFilesTest.java  | 366 ++++++++--
 .../options/PipelineOptionsValidatorTest.java   |  44 ++
 .../sdk/options/ProxyInvocationHandlerTest.java |  19 +
 .../sdk/runners/TransformHierarchyTest.java     | 197 +++++
 .../sdk/testing/PCollectionViewTesting.java     |   8 +
 .../apache/beam/sdk/transforms/CombineTest.java | 365 ++++++----
 .../beam/sdk/transforms/DoFnTesterTest.java     |  32 +
 .../beam/sdk/transforms/GroupByKeyTest.java     |  39 +
 .../apache/beam/sdk/transforms/ParDoTest.java   | 165 +++++
 .../beam/sdk/transforms/SplittableDoFnTest.java | 155 +++-
 .../transforms/reflect/DoFnInvokersTest.java    |  93 ++-
 .../DoFnSignaturesProcessElementTest.java       |   2 +-
 .../DoFnSignaturesSplittableDoFnTest.java       |  83 ++-
 .../transforms/reflect/DoFnSignaturesTest.java  |  14 +
 .../splittabledofn/OffsetRangeTrackerTest.java  |   1 +
 .../windowing/SlidingWindowsTest.java           |  30 +-
 .../google-cloud-platform-core/pom.xml          |   2 +-
 .../java/org/apache/beam/sdk/util/GcsUtil.java  |   2 +-
 .../sdk/util/RetryHttpRequestInitializer.java   | 147 ++--
 .../extensions/gcp/GcpCoreApiSurfaceTest.java   |  48 +-
 .../util/RetryHttpRequestInitializerTest.java   |  31 +-
 sdks/java/extensions/jackson/pom.xml            |   2 +-
 sdks/java/extensions/join-library/pom.xml       |   2 +-
 sdks/java/extensions/pom.xml                    |   2 +-
 sdks/java/extensions/protobuf/pom.xml           |   2 +-
 sdks/java/extensions/sorter/pom.xml             |   8 +-
 sdks/java/harness/pom.xml                       |  18 +-
 .../harness/control/ProcessBundleHandler.java   | 295 ++------
 .../fn/harness/control/RegisterHandler.java     |   2 +-
 .../beam/runners/core/BeamFnDataReadRunner.java |  70 +-
 .../runners/core/BeamFnDataWriteRunner.java     |  67 +-
 .../beam/runners/core/BoundedSourceRunner.java  |  74 +-
 .../beam/runners/core/FnApiDoFnRunner.java      | 547 ++++++++++++++
 .../runners/core/PTransformRunnerFactory.java   |  81 +++
 .../control/ProcessBundleHandlerTest.java       | 521 ++------------
 .../fn/harness/control/RegisterHandlerTest.java |   8 +-
 .../runners/core/BeamFnDataReadRunnerTest.java  | 112 ++-
 .../runners/core/BeamFnDataWriteRunnerTest.java | 120 +++-
 .../runners/core/BoundedSourceRunnerTest.java   | 124 +++-
 .../beam/runners/core/FnApiDoFnRunnerTest.java  | 210 ++++++
 sdks/java/io/amqp/pom.xml                       | 100 +++
 .../org/apache/beam/sdk/io/amqp/AmqpIO.java     | 399 +++++++++++
 .../beam/sdk/io/amqp/AmqpMessageCoder.java      |  79 ++
 .../amqp/AmqpMessageCoderProviderRegistrar.java |  44 ++
 .../apache/beam/sdk/io/amqp/package-info.java   |  22 +
 .../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 148 ++++
 .../beam/sdk/io/amqp/AmqpMessageCoderTest.java  |  89 +++
 sdks/java/io/cassandra/pom.xml                  |   2 +-
 .../beam/sdk/io/cassandra/CassandraIO.java      |   2 +-
 sdks/java/io/common/pom.xml                     |   2 +-
 .../sdk/io/common/IOTestPipelineOptions.java    |   6 +-
 sdks/java/io/elasticsearch/pom.xml              |  10 +-
 .../sdk/io/elasticsearch/ElasticsearchIO.java   |  17 +-
 .../elasticsearch/ElasticSearchIOTestUtils.java |  81 ++-
 .../sdk/io/elasticsearch/ElasticsearchIOIT.java |  14 +-
 .../io/elasticsearch/ElasticsearchIOTest.java   |  36 +-
 .../elasticsearch/ElasticsearchTestDataSet.java |  37 +-
 sdks/java/io/google-cloud-platform/pom.xml      |  14 +-
 .../beam/sdk/io/gcp/bigquery/BatchLoads.java    |   2 +
 .../io/gcp/bigquery/DynamicDestinations.java    |  29 +-
 .../io/gcp/bigquery/GenerateShardedTable.java   |   1 +
 .../beam/sdk/io/gcp/bigquery/ShardedKey.java    |  67 --
 .../sdk/io/gcp/bigquery/ShardedKeyCoder.java    |  74 --
 .../sdk/io/gcp/bigquery/StreamingWriteFn.java   |   1 +
 .../io/gcp/bigquery/StreamingWriteTables.java   |   2 +
 .../sdk/io/gcp/bigquery/TagWithUniqueIds.java   |   1 +
 .../io/gcp/bigquery/WriteBundlesToFiles.java    |   2 +
 .../bigquery/WriteGroupedRecordsToFiles.java    |   1 +
 .../sdk/io/gcp/bigquery/WritePartition.java     |   1 +
 .../beam/sdk/io/gcp/bigquery/WriteTables.java   |   1 +
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java    |   8 +-
 .../io/gcp/bigtable/BigtableServiceImpl.java    |   9 +-
 .../sdk/io/gcp/datastore/AdaptiveThrottler.java | 103 +++
 .../beam/sdk/io/gcp/datastore/DatastoreV1.java  | 149 +++-
 .../sdk/io/gcp/datastore/MovingAverage.java     |  50 ++
 .../sdk/io/gcp/spanner/AbstractSpannerFn.java   |  58 ++
 .../sdk/io/gcp/spanner/CreateTransactionFn.java |  51 ++
 .../beam/sdk/io/gcp/spanner/MutationGroup.java  |  67 ++
 .../io/gcp/spanner/MutationSizeEstimator.java   |   9 +
 .../sdk/io/gcp/spanner/NaiveSpannerReadFn.java  |  65 ++
 .../beam/sdk/io/gcp/spanner/SpannerConfig.java  | 137 ++++
 .../beam/sdk/io/gcp/spanner/SpannerIO.java      | 616 +++++++++++-----
 .../sdk/io/gcp/spanner/SpannerWriteGroupFn.java | 125 ++++
 .../beam/sdk/io/gcp/spanner/Transaction.java    |  33 +
 .../beam/sdk/io/gcp/GcpApiSurfaceTest.java      |  10 +
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     |   2 +
 .../sdk/io/gcp/bigtable/BigtableReadIT.java     |   5 +-
 .../io/gcp/bigtable/BigtableTestOptions.java    |   5 -
 .../sdk/io/gcp/bigtable/BigtableWriteIT.java    |   4 +-
 .../io/gcp/datastore/AdaptiveThrottlerTest.java | 111 +++
 .../sdk/io/gcp/datastore/DatastoreV1Test.java   |  92 ++-
 .../beam/sdk/io/gcp/datastore/V1TestUtil.java   |   2 +-
 .../sdk/io/gcp/spanner/FakeServiceFactory.java  |  82 +++
 .../gcp/spanner/MutationSizeEstimatorTest.java  |  12 +
 .../beam/sdk/io/gcp/spanner/RandomUtils.java    |  41 ++
 .../sdk/io/gcp/spanner/SpannerIOReadTest.java   | 281 ++++++++
 .../beam/sdk/io/gcp/spanner/SpannerIOTest.java  | 244 -------
 .../sdk/io/gcp/spanner/SpannerIOWriteTest.java  | 258 +++++++
 .../beam/sdk/io/gcp/spanner/SpannerReadIT.java  | 166 +++++
 .../beam/sdk/io/gcp/spanner/SpannerWriteIT.java |  26 +-
 sdks/java/io/hadoop-common/pom.xml              |   2 +-
 sdks/java/io/hadoop-file-system/pom.xml         |  33 +-
 sdks/java/io/hadoop/input-format/pom.xml        |   2 +-
 .../hadoop/inputformat/HadoopInputFormatIO.java |   2 +-
 sdks/java/io/hadoop/jdk1.8-tests/pom.xml        |   4 +-
 .../inputformat/HIFIOWithElasticTest.java       |  11 +-
 sdks/java/io/hadoop/pom.xml                     |   2 +-
 sdks/java/io/hbase/pom.xml                      |  26 +-
 .../io/hbase/HBaseCoderProviderRegistrar.java   |  40 ++
 .../org/apache/beam/sdk/io/hbase/HBaseIO.java   |  48 +-
 .../beam/sdk/io/hbase/HBaseMutationCoder.java   |  42 ++
 .../hbase/HBaseCoderProviderRegistrarTest.java  |  45 ++
 .../apache/beam/sdk/io/hbase/HBaseIOTest.java   |  49 +-
 sdks/java/io/hcatalog/pom.xml                   | 175 +++++
 .../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 492 +++++++++++++
 .../beam/sdk/io/hcatalog/package-info.java      |  22 +
 .../io/hcatalog/EmbeddedMetastoreService.java   |  87 +++
 .../beam/sdk/io/hcatalog/HCatalogIOTest.java    | 277 ++++++++
 .../sdk/io/hcatalog/HCatalogIOTestUtils.java    | 108 +++
 .../hcatalog/src/test/resources/hive-site.xml   | 301 ++++++++
 sdks/java/io/jdbc/pom.xml                       |   4 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java     |   2 +-
 sdks/java/io/jms/pom.xml                        |   2 +-
 .../java/org/apache/beam/sdk/io/jms/JmsIO.java  |   2 +-
 sdks/java/io/kafka/pom.xml                      |   2 +-
 .../org/apache/beam/sdk/io/kafka/KafkaIO.java   | 132 ++--
 .../apache/beam/sdk/io/kafka/KafkaIOTest.java   |  30 +
 sdks/java/io/kinesis/pom.xml                    |   2 +-
 .../sdk/io/kinesis/CheckpointGenerator.java     |   6 +-
 .../beam/sdk/io/kinesis/CustomOptional.java     | 111 +--
 .../io/kinesis/DynamicCheckpointGenerator.java  |  52 +-
 .../sdk/io/kinesis/GetKinesisRecordsResult.java |  49 +-
 .../sdk/io/kinesis/KinesisClientProvider.java   |   4 +-
 .../apache/beam/sdk/io/kinesis/KinesisIO.java   | 281 ++++----
 .../beam/sdk/io/kinesis/KinesisReader.java      | 206 +++---
 .../sdk/io/kinesis/KinesisReaderCheckpoint.java |  97 +--
 .../beam/sdk/io/kinesis/KinesisRecord.java      | 177 ++---
 .../beam/sdk/io/kinesis/KinesisRecordCoder.java |  68 +-
 .../beam/sdk/io/kinesis/KinesisSource.java      | 147 ++--
 .../beam/sdk/io/kinesis/RecordFilter.java       |  18 +-
 .../apache/beam/sdk/io/kinesis/RoundRobin.java  |  37 +-
 .../beam/sdk/io/kinesis/ShardCheckpoint.java    | 241 +++----
 .../sdk/io/kinesis/ShardRecordsIterator.java    | 106 +--
 .../sdk/io/kinesis/SimplifiedKinesisClient.java | 215 +++---
 .../beam/sdk/io/kinesis/StartingPoint.java      |  84 +--
 .../io/kinesis/StaticCheckpointGenerator.java   |  27 +-
 .../io/kinesis/TransientKinesisException.java   |   7 +-
 .../beam/sdk/io/kinesis/AmazonKinesisMock.java  | 539 +++++++-------
 .../beam/sdk/io/kinesis/CustomOptionalTest.java |  27 +-
 .../kinesis/DynamicCheckpointGeneratorTest.java |  33 +-
 .../sdk/io/kinesis/KinesisMockReadTest.java     |  97 +--
 .../io/kinesis/KinesisReaderCheckpointTest.java |  52 +-
 .../beam/sdk/io/kinesis/KinesisReaderIT.java    | 127 ++--
 .../beam/sdk/io/kinesis/KinesisReaderTest.java  | 166 ++---
 .../sdk/io/kinesis/KinesisRecordCoderTest.java  |  34 +-
 .../beam/sdk/io/kinesis/KinesisTestOptions.java |  43 +-
 .../beam/sdk/io/kinesis/KinesisUploader.java    |  70 +-
 .../beam/sdk/io/kinesis/RecordFilterTest.java   |  52 +-
 .../beam/sdk/io/kinesis/RoundRobinTest.java     |  42 +-
 .../sdk/io/kinesis/ShardCheckpointTest.java     | 203 +++---
 .../io/kinesis/ShardRecordsIteratorTest.java    | 216 +++---
 .../io/kinesis/SimplifiedKinesisClientTest.java | 351 ++++-----
 sdks/java/io/mongodb/pom.xml                    |   2 +-
 .../beam/sdk/io/mongodb/MongoDbGridFSIO.java    |   2 +-
 .../apache/beam/sdk/io/mongodb/MongoDbIO.java   | 317 +++++++--
 .../beam/sdk/io/mongodb/MongoDbIOTest.java      |  37 +
 sdks/java/io/mqtt/pom.xml                       |   2 +-
 .../org/apache/beam/sdk/io/mqtt/MqttIO.java     |   2 +-
 sdks/java/io/pom.xml                            |  35 +-
 sdks/java/io/xml/pom.xml                        |   2 +-
 .../java/org/apache/beam/sdk/io/xml/XmlIO.java  |   4 +-
 .../org/apache/beam/sdk/io/xml/XmlSink.java     |  21 +-
 .../org/apache/beam/sdk/io/xml/XmlSinkTest.java |   4 +-
 sdks/java/java8tests/pom.xml                    |   2 +-
 sdks/java/javadoc/pom.xml                       |  19 +-
 .../maven-archetypes/examples-java8/pom.xml     |   2 +-
 .../main/resources/archetype-resources/pom.xml  |   1 -
 sdks/java/maven-archetypes/examples/pom.xml     |   2 +-
 .../main/resources/archetype-resources/pom.xml  |   1 -
 sdks/java/maven-archetypes/pom.xml              |   2 +-
 sdks/java/maven-archetypes/starter/pom.xml      |   2 +-
 .../resources/projects/basic/reference/pom.xml  |   2 +-
 sdks/java/pom.xml                               |   2 +-
 sdks/pom.xml                                    |   2 +-
 sdks/python/apache_beam/coders/coder_impl.py    |   4 +
 sdks/python/apache_beam/coders/coders.py        |   7 +-
 .../apache_beam/coders/coders_test_common.py    |   8 +
 .../examples/snippets/snippets_test.py          |  16 +
 .../apache_beam/examples/streaming_wordcount.py |  25 +-
 .../apache_beam/examples/windowed_wordcount.py  |  93 +++
 sdks/python/apache_beam/io/filesystem.py        |  22 +-
 sdks/python/apache_beam/io/gcp/gcsio.py         |  10 +-
 sdks/python/apache_beam/io/gcp/pubsub.py        | 180 +++--
 sdks/python/apache_beam/io/gcp/pubsub_test.py   | 101 ++-
 .../io/gcp/tests/bigquery_matcher.py            |   6 +-
 .../io/gcp/tests/bigquery_matcher_test.py       |   2 +-
 sdks/python/apache_beam/io/range_trackers.py    | 130 ----
 .../apache_beam/io/range_trackers_test.py       | 186 -----
 .../apache_beam/options/pipeline_options.py     |  35 +-
 .../options/pipeline_options_test.py            |  39 +-
 .../apache_beam/options/value_provider_test.py  |  93 +--
 sdks/python/apache_beam/pipeline.py             | 230 +++++-
 sdks/python/apache_beam/pipeline_test.py        |  53 ++
 sdks/python/apache_beam/portability/__init__.py |  18 +
 .../apache_beam/portability/api/__init__.py     |  21 +
 sdks/python/apache_beam/pvalue.py               |   2 +-
 sdks/python/apache_beam/runners/api/__init__.py |  21 -
 .../runners/dataflow/dataflow_runner.py         |  91 ++-
 .../runners/dataflow/dataflow_runner_test.py    |  24 +-
 .../runners/dataflow/internal/apiclient.py      |  35 +-
 .../runners/dataflow/internal/apiclient_test.py |  29 +-
 .../runners/dataflow/internal/dependency.py     |  69 +-
 .../runners/dataflow/native_io/iobase_test.py   |  39 +-
 .../dataflow/native_io/streaming_create.py      |  72 ++
 .../runners/dataflow/ptransform_overrides.py    |  52 ++
 .../runners/direct/bundle_factory.py            |   2 +-
 .../apache_beam/runners/direct/direct_runner.py | 108 +++
 .../runners/direct/evaluation_context.py        |  73 +-
 .../apache_beam/runners/direct/executor.py      | 135 ++--
 .../runners/direct/transform_evaluator.py       | 447 +++++++++++-
 .../runners/direct/transform_result.py          |  41 --
 sdks/python/apache_beam/runners/direct/util.py  |  67 ++
 .../runners/direct/watermark_manager.py         | 100 ++-
 .../apache_beam/runners/pipeline_context.py     |  19 +-
 .../runners/portability/fn_api_runner.py        | 306 ++++----
 .../runners/portability/fn_api_runner_test.py   |  31 +-
 .../runners/worker/bundle_processor.py          | 426 +++++++++++
 .../apache_beam/runners/worker/data_plane.py    |  28 +-
 .../runners/worker/data_plane_test.py           |   2 +-
 .../apache_beam/runners/worker/log_handler.py   |   2 +-
 .../runners/worker/log_handler_test.py          |   2 +-
 .../runners/worker/operation_specs.py           |   9 +-
 .../apache_beam/runners/worker/operations.py    |   1 +
 .../apache_beam/runners/worker/sdk_worker.py    | 370 +---------
 .../runners/worker/sdk_worker_main.py           |   2 +-
 .../runners/worker/sdk_worker_test.py           |  95 +--
 sdks/python/apache_beam/testing/test_stream.py  |   5 +
 .../apache_beam/testing/test_stream_test.py     |  68 ++
 sdks/python/apache_beam/transforms/combiners.py |   8 +
 .../apache_beam/transforms/combiners_test.py    |   7 +-
 sdks/python/apache_beam/transforms/core.py      | 102 +--
 .../python/apache_beam/transforms/ptransform.py |  43 +-
 sdks/python/apache_beam/transforms/trigger.py   |  28 +-
 sdks/python/apache_beam/transforms/window.py    |   4 +-
 .../apache_beam/typehints/trivial_inference.py  |   3 +-
 .../typehints/trivial_inference_test.py         |   7 +
 sdks/python/apache_beam/utils/plugin.py         |  42 ++
 sdks/python/apache_beam/utils/timestamp.py      |   5 +
 sdks/python/apache_beam/utils/urns.py           |   2 +-
 sdks/python/apache_beam/version.py              |   2 +-
 sdks/python/gen_protos.py                       |   2 +-
 sdks/python/pom.xml                             |   2 +-
 sdks/python/run_pylint.sh                       |   2 +-
 sdks/python/setup.py                            |   5 +-
 462 files changed, 21718 insertions(+), 10754 deletions(-)
----------------------------------------------------------------------



Mime
View raw message