beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [50/50] incubator-beam git commit: Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner
Date Wed, 23 Nov 2016 06:52:46 GMT
Merge branch 'master' of https://github.com/apache/incubator-beam into gearpump-runner


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/28124050
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/28124050
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/28124050

Branch: refs/heads/gearpump-runner
Commit: 28124050d5598acf8c9df621551ed2d84dc643fd
Parents: 68363d0 dd85cad
Author: manuzhang <owenzhang1990@gmail.com>
Authored: Wed Nov 23 12:32:37 2016 +0800
Committer: manuzhang <owenzhang1990@gmail.com>
Committed: Wed Nov 23 12:32:37 2016 +0800

----------------------------------------------------------------------
 README.md                                       |   4 +-
 pom.xml                                         |  34 +-
 runners/apex/pom.xml                            |   3 +-
 .../translation/ParDoBoundMultiTranslator.java  |  67 ++-
 .../apex/translation/ParDoBoundTranslator.java  |  46 +-
 .../operators/ApexGroupByKeyOperator.java       |  26 +-
 runners/core-java/pom.xml                       |   2 +-
 .../core/GroupAlsoByWindowViaWindowSetDoFn.java |   3 +-
 .../GroupAlsoByWindowsViaOutputBufferDoFn.java  |   5 +-
 .../beam/runners/core/OutputWindowedValue.java  |  45 ++
 .../beam/runners/core/PaneInfoTracker.java      |   3 -
 .../runners/core/ReduceFnContextFactory.java    |  83 +++-
 .../beam/runners/core/ReduceFnRunner.java       |  40 +-
 .../beam/runners/core/SimpleDoFnRunner.java     | 142 ++++--
 .../beam/runners/core/SimpleOldDoFnRunner.java  |  21 +-
 .../beam/runners/core/SplittableParDo.java      |  35 +-
 .../beam/runners/core/SystemReduceFn.java       |   3 -
 .../apache/beam/runners/core/WatermarkHold.java |   3 -
 .../core/WindowingInternalsAdapters.java        |  76 ++++
 .../AfterDelayFromFirstElementStateMachine.java |   5 -
 .../core/triggers/AfterPaneStateMachine.java    |   3 -
 .../core/triggers/TriggerStateMachines.java     |  15 -
 .../beam/runners/core/ReduceFnTester.java       |  81 +---
 runners/direct-java/pom.xml                     |   2 +-
 .../direct/BoundedReadEvaluatorFactory.java     | 110 ++++-
 .../beam/runners/direct/DirectRunner.java       |  18 +-
 .../beam/runners/direct/EmptyInputProvider.java |  22 +-
 .../direct/ExecutorServiceParallelExecutor.java |  12 +-
 .../GroupAlsoByWindowEvaluatorFactory.java      |  92 ++--
 .../runners/direct/ParDoEvaluatorFactory.java   |  56 +--
 .../direct/ParDoMultiEvaluatorHooks.java        |  55 ---
 .../direct/ParDoMultiOverrideFactory.java       |  51 +++
 .../runners/direct/ParDoOverrideFactory.java    |  53 ---
 .../direct/ParDoSingleEvaluatorHooks.java       |  58 ---
 .../ParDoSingleViaMultiOverrideFactory.java     |  66 +++
 .../beam/runners/direct/RootInputProvider.java  |  16 +-
 .../runners/direct/RootProviderRegistry.java    |  19 +-
 .../runners/direct/StepTransformResult.java     |   6 +
 .../direct/TestStreamEvaluatorFactory.java      |  23 +-
 .../direct/TransformEvaluatorRegistry.java      |  12 +-
 .../beam/runners/direct/TransformExecutor.java  |  28 +-
 .../direct/UnboundedReadEvaluatorFactory.java   |  32 +-
 .../direct/BoundedReadEvaluatorFactoryTest.java | 186 +++++++-
 .../direct/ParDoMultiEvaluatorHooksTest.java    | 439 -------------------
 .../direct/ParDoSingleEvaluatorHooksTest.java   | 335 --------------
 .../runners/direct/TransformExecutorTest.java   |  43 --
 runners/flink/runner/pom.xml                    |   1 +
 .../FlinkBatchTransformTranslators.java         |  34 +-
 .../FlinkStreamingTransformTranslators.java     |  38 +-
 .../functions/FlinkDoFnFunction.java            |  21 +-
 .../FlinkMergingNonShuffleReduceFunction.java   |  14 +-
 .../FlinkMergingPartialReduceFunction.java      |  14 +-
 .../functions/FlinkMergingReduceFunction.java   |  12 +-
 .../functions/FlinkMultiOutputDoFnFunction.java |  14 +-
 .../FlinkMultiOutputProcessContext.java         |  94 +---
 .../functions/FlinkPartialReduceFunction.java   |  14 +-
 .../functions/FlinkProcessContext.java          | 330 --------------
 .../functions/FlinkProcessContextBase.java      | 278 ++++++++++++
 .../functions/FlinkReduceFunction.java          |  14 +-
 .../FlinkSingleOutputProcessContext.java        |  69 +++
 runners/google-cloud-dataflow-java/pom.xml      |   2 +-
 .../beam/runners/dataflow/DataflowRunner.java   |   2 +-
 .../dataflow/util/DataflowTransport.java        |  10 +-
 .../options/DataflowPipelineOptionsTest.java    |   6 +-
 .../runners/dataflow/util/PackageUtilTest.java  |   2 +-
 runners/spark/pom.xml                           |   4 +-
 .../runners/spark/SparkPipelineOptions.java     |  10 +-
 .../apache/beam/runners/spark/SparkRunner.java  |   9 +-
 .../spark/translation/BoundedDataset.java       | 114 +++++
 .../beam/runners/spark/translation/Dataset.java |  34 ++
 .../runners/spark/translation/DoFnFunction.java |  10 +-
 .../spark/translation/EvaluationContext.java    | 257 +++++------
 .../spark/translation/MultiDoFnFunction.java    |  21 +-
 .../spark/translation/SparkProcessContext.java  |  81 ++--
 .../spark/translation/SparkRuntimeContext.java  |   4 +-
 .../spark/translation/TransformTranslator.java  | 122 +++---
 .../SparkRunnerStreamingContextFactory.java     |   8 +-
 .../streaming/StreamingEvaluationContext.java   | 272 ------------
 .../streaming/StreamingTransformTranslator.java | 135 +++---
 .../translation/streaming/UnboundedDataset.java | 103 +++++
 .../runners/spark/ProvidedSparkContextTest.java |  24 +-
 .../metrics/sink/NamedAggregatorsTest.java      |  13 +-
 .../beam/runners/spark/io/AvroPipelineTest.java |  11 +-
 .../beam/runners/spark/io/NumShardsTest.java    |  11 +-
 .../io/hadoop/HadoopFileFormatPipelineTest.java |  11 +-
 .../spark/translation/SideEffectsTest.java      |  41 +-
 .../streaming/EmptyStreamAssertionTest.java     |   8 +-
 .../streaming/FlattenStreamingTest.java         |   9 +-
 .../streaming/KafkaStreamingTest.java           |  16 +-
 .../ResumeFromCheckpointStreamingTest.java      |   8 +-
 .../streaming/SimpleStreamingWordCountTest.java |  11 +-
 .../streaming/utils/PAssertStreaming.java       |   8 +-
 .../utils/SparkTestPipelineOptions.java         |  42 ++
 .../SparkTestPipelineOptionsForStreaming.java   |  43 ++
 .../utils/TestOptionsForStreaming.java          |  55 ---
 .../src/main/resources/beam/findbugs-filter.xml | 172 +++++++-
 sdks/java/core/pom.xml                          |   2 +-
 .../main/java/org/apache/beam/sdk/Pipeline.java |  17 -
 .../org/apache/beam/sdk/coders/AvroCoder.java   |   5 -
 .../apache/beam/sdk/coders/InstantCoder.java    |   4 -
 .../org/apache/beam/sdk/io/FileBasedSink.java   | 197 +--------
 .../beam/sdk/metrics/MetricsContainer.java      |  13 -
 .../beam/sdk/metrics/MetricsEnvironment.java    |  60 ++-
 .../sdk/options/PipelineOptionsFactory.java     |  32 +-
 .../apache/beam/sdk/runners/PipelineRunner.java |   2 +-
 .../org/apache/beam/sdk/testing/PAssert.java    |   3 -
 .../beam/sdk/testing/SerializableMatchers.java  |   3 -
 .../apache/beam/sdk/testing/TestPipeline.java   |   2 +-
 .../beam/sdk/testing/UsesStatefulParDo.java     |  25 ++
 .../org/apache/beam/sdk/transforms/DoFn.java    | 109 +----
 .../beam/sdk/transforms/DoFnAdapters.java       |  42 +-
 .../apache/beam/sdk/transforms/DoFnTester.java  |  18 +-
 .../org/apache/beam/sdk/transforms/OldDoFn.java |   7 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |   2 +-
 .../beam/sdk/transforms/WithTimestamps.java     |   5 +-
 .../reflect/ByteBuddyDoFnInvokerFactory.java    | 325 +++++++-------
 .../reflect/ByteBuddyOnTimerInvokerFactory.java |  42 +-
 .../sdk/transforms/reflect/DoFnInvoker.java     | 123 +++++-
 .../sdk/transforms/reflect/DoFnInvokers.java    |  14 +-
 .../sdk/transforms/reflect/DoFnSignature.java   |  52 ++-
 .../sdk/transforms/reflect/DoFnSignatures.java  |  61 ++-
 .../sdk/transforms/reflect/OnTimerInvoker.java  |  10 +-
 ...AttemptAndTimeBoundedExponentialBackOff.java |   3 -
 .../sdk/util/ExposedByteArrayInputStream.java   |   2 -
 .../sdk/util/ExposedByteArrayOutputStream.java  |   3 -
 .../beam/sdk/util/FileIOChannelFactory.java     |  67 ++-
 .../sdk/util/FileIOChannelFactoryRegistrar.java |  38 ++
 .../beam/sdk/util/GcpCredentialFactory.java     |  13 +-
 .../beam/sdk/util/GcsIOChannelFactory.java      |  20 +-
 .../sdk/util/GcsIOChannelFactoryRegistrar.java  |  38 ++
 .../apache/beam/sdk/util/IOChannelFactory.java  |  22 +
 .../sdk/util/IOChannelFactoryRegistrar.java     |  48 ++
 .../apache/beam/sdk/util/IOChannelUtils.java    | 134 +++++-
 .../beam/sdk/util/NoopCredentialFactory.java    |  31 +-
 .../sdk/util/NullCredentialInitializer.java     |  62 +++
 .../apache/beam/sdk/util/TimerInternals.java    |  13 +-
 .../org/apache/beam/sdk/util/Transport.java     |   9 +-
 .../beam/sdk/util/WindowingInternals.java       |  14 +-
 .../java/org/apache/beam/sdk/util/ZipFiles.java |   9 -
 .../beam/sdk/util/common/ReflectHelpers.java    |  29 ++
 .../beam/sdk/util/state/StateContexts.java      |  20 +-
 .../beam/sdk/util/state/StateMerging.java       |   3 -
 .../java/org/apache/beam/sdk/io/AvroIOTest.java |   2 +-
 .../apache/beam/sdk/io/FileBasedSourceTest.java |   2 +-
 .../java/org/apache/beam/sdk/io/TextIOTest.java |   2 +-
 .../sdk/metrics/MetricsEnvironmentTest.java     |   8 +-
 .../apache/beam/sdk/metrics/MetricsTest.java    |   6 +-
 .../sdk/options/PipelineOptionsFactoryTest.java |  34 --
 .../apache/beam/sdk/transforms/OldDoFnTest.java |  11 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   |  55 ++-
 .../transforms/reflect/DoFnInvokersTest.java    | 228 ++++++++--
 .../DoFnSignaturesProcessElementTest.java       |  22 -
 .../DoFnSignaturesSplittableDoFnTest.java       |   2 +-
 .../transforms/reflect/DoFnSignaturesTest.java  |  25 +-
 .../transforms/reflect/OnTimerInvokersTest.java |   7 +-
 .../testhelper/DoFnInvokersTestHelper.java      | 137 ++++++
 .../util/FileIOChannelFactoryRegistrarTest.java |  44 ++
 .../beam/sdk/util/FileIOChannelFactoryTest.java |  10 +-
 .../util/GcsIOChannelFactoryRegistrarTest.java  |  44 ++
 .../beam/sdk/util/GcsIOChannelFactoryTest.java  |   2 +-
 .../beam/sdk/util/IOChannelUtilsTest.java       |  39 ++
 .../sdk/util/common/ReflectHelpersTest.java     |  33 ++
 sdks/java/io/google-cloud-platform/pom.xml      |  14 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    |   4 +
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     |   6 +-
 sdks/java/io/hdfs/pom.xml                       |   2 +-
 sdks/java/io/jdbc/pom.xml                       |   2 +-
 sdks/java/io/jms/pom.xml                        |   2 +-
 sdks/java/io/kafka/pom.xml                      |   2 +-
 sdks/java/io/kinesis/pom.xml                    |   2 +-
 sdks/java/io/mongodb/pom.xml                    |  15 +-
 .../beam/sdk/io/mongodb/MongoDbGridFSIO.java    | 308 +++++++++++--
 .../apache/beam/sdk/io/mongodb/MongoDbIO.java   |   8 +-
 .../sdk/io/mongodb/MongoDBGridFSIOTest.java     | 121 ++++-
 .../beam/sdk/io/mongodb/MongoDbIOTest.java      |  38 +-
 sdks/java/maven-archetypes/examples/pom.xml     |  42 ++
 .../main/resources/archetype-resources/pom.xml  |  14 +
 sdks/java/maven-archetypes/starter/pom.xml      |   6 +
 .../main/resources/archetype-resources/pom.xml  |  14 +
 .../src/main/java/StarterPipeline.java          |  18 +-
 .../resources/projects/basic/reference/pom.xml  |  14 +
 .../src/main/java/it/pkg/StarterPipeline.java   |  18 +-
 sdks/java/microbenchmarks/README.md             |  42 --
 sdks/java/microbenchmarks/pom.xml               | 113 -----
 .../coders/AvroCoderBenchmark.java              | 119 -----
 .../coders/ByteArrayCoderBenchmark.java         |  64 ---
 .../coders/CoderBenchmarking.java               |  41 --
 .../coders/StringUtf8CoderBenchmark.java        |  70 ---
 .../microbenchmarks/coders/package-info.java    |  22 -
 .../transforms/DoFnInvokersBenchmark.java       | 231 ----------
 .../transforms/package-info.java                |  22 -
 sdks/java/pom.xml                               |   1 -
 192 files changed, 4606 insertions(+), 4350 deletions(-)
----------------------------------------------------------------------



Mime
View raw message