beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [48/50] [abbrv] incubator-beam git commit: Merge remote-tracking branch 'upstream/master' into gearpump-runner-sync
Date Tue, 06 Dec 2016 16:41:38 GMT
Merge remote-tracking branch 'upstream/master' into gearpump-runner-sync


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/86414c08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/86414c08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/86414c08

Branch: refs/heads/gearpump-runner
Commit: 86414c08eb70c8975dac7b4750ca02b8b9cf875b
Parents: 2812405 ef9871c
Author: manuzhang <owenzhang1990@gmail.com>
Authored: Mon Dec 5 11:30:31 2016 +0800
Committer: manuzhang <owenzhang1990@gmail.com>
Committed: Mon Dec 5 11:30:31 2016 +0800

----------------------------------------------------------------------
 .gitignore                                      |   4 +
 .jenkins/common_job_properties.groovy           | 166 ++++++
 ...job_beam_PostCommit_Java_MavenInstall.groovy |  42 ++
 ...ommit_Java_RunnableOnService_Dataflow.groovy |  39 ++
 ...stCommit_Java_RunnableOnService_Flink.groovy |  38 ++
 ...ommit_Java_RunnableOnService_Gearpump.groovy |  41 ++
 ...stCommit_Java_RunnableOnService_Spark.groovy |  38 ++
 .../job_beam_PostCommit_Python_Verify.groovy    |  37 ++
 .../job_beam_PreCommit_Java_MavenInstall.groovy |  42 ++
 .../job_beam_Release_NightlySnapshot.groovy     |  46 ++
 .jenkins/job_seed.groovy                        |  47 ++
 .travis.yml                                     |   2 +-
 .travis/README.md                               |   2 +-
 .../org/apache/beam/examples/complete/README.md |  14 +-
 .../beam/examples/cookbook/DeDupExample.java    |  96 ---
 .../beam/examples/cookbook/DistinctExample.java |  96 +++
 .../org/apache/beam/examples/cookbook/README.md |  16 +-
 .../org/apache/beam/examples/WordCountIT.java   |   2 +-
 pom.xml                                         |  10 +-
 runners/apex/pom.xml                            |  11 +-
 .../translation/ApexPipelineTranslator.java     |  12 +-
 .../apex/translation/TranslationContext.java    |   6 +-
 .../operators/ApexGroupByKeyOperator.java       |  14 +-
 .../core/ElementAndRestrictionCoder.java        |   8 +
 .../runners/core/GBKIntoKeyedWorkItems.java     |  55 --
 .../GroupAlsoByWindowsViaOutputBufferDoFn.java  |   6 +-
 .../core/GroupByKeyViaGroupByKeyOnly.java       |  22 +-
 .../runners/core/PerKeyCombineFnRunners.java    | 263 +++++++++
 .../core/PushbackSideInputDoFnRunner.java       |  20 +-
 .../beam/runners/core/SimpleDoFnRunner.java     |  76 ++-
 .../beam/runners/core/SimpleOldDoFnRunner.java  |  16 -
 .../beam/runners/core/SplittableParDo.java      | 378 +++++++++---
 .../core/GroupAlsoByWindowsProperties.java      | 590 +++++++++++--------
 .../core/PushbackSideInputDoFnRunnerTest.java   |  18 +-
 .../beam/runners/core/ReduceFnRunnerTest.java   |   5 +-
 .../beam/runners/core/SplittableParDoTest.java  | 132 ++++-
 .../beam/runners/core/WindowMatchers.java       | 204 +++++++
 .../beam/runners/core/WindowMatchersTest.java   |  82 +++
 .../direct/AbstractModelEnforcement.java        |   2 +-
 .../direct/BoundedReadEvaluatorFactory.java     |   8 +-
 .../beam/runners/direct/CommittedResult.java    |   2 +-
 .../beam/runners/direct/CompletionCallback.java |   2 +-
 .../direct/ConsumerTrackingPipelineVisitor.java | 173 ------
 ...ectGBKIntoKeyedWorkItemsOverrideFactory.java |  41 +-
 .../apache/beam/runners/direct/DirectGraph.java |  89 +++
 .../beam/runners/direct/DirectGraphVisitor.java | 145 +++++
 .../beam/runners/direct/DirectGroupByKey.java   |   2 +-
 .../beam/runners/direct/DirectRunner.java       |  48 +-
 .../runners/direct/DirectTimerInternals.java    |   4 +-
 .../runners/direct/DisplayDataValidator.java    |   6 +-
 .../runners/direct/DoFnLifecycleManager.java    |   4 +-
 ...ecycleManagerRemovingTransformEvaluator.java |   2 +-
 .../runners/direct/EmptyTransformEvaluator.java |   4 +-
 .../beam/runners/direct/EvaluationContext.java  |  87 ++-
 .../direct/ExecutorServiceParallelExecutor.java |  17 +-
 .../runners/direct/FlattenEvaluatorFactory.java |  10 +-
 .../GroupAlsoByWindowEvaluatorFactory.java      |   5 +-
 .../direct/GroupByKeyOnlyEvaluatorFactory.java  |   2 +-
 .../ImmutabilityCheckingBundleFactory.java      |  21 +-
 .../direct/ImmutabilityEnforcementFactory.java  |   2 +-
 .../direct/KeyedPValueTrackingVisitor.java      |  12 +-
 .../beam/runners/direct/ModelEnforcement.java   |   2 +-
 .../beam/runners/direct/ParDoEvaluator.java     |  33 +-
 .../runners/direct/ParDoEvaluatorFactory.java   |  94 ++-
 .../direct/ParDoMultiOverrideFactory.java       |  76 ++-
 .../ParDoSingleViaMultiOverrideFactory.java     |   6 +-
 .../direct/PassthroughTransformEvaluator.java   |   4 +-
 ...littableProcessElementsEvaluatorFactory.java | 144 +++++
 .../direct/StatefulParDoEvaluatorFactory.java   | 256 ++++++++
 .../runners/direct/StepTransformResult.java     |  38 +-
 .../direct/TestStreamEvaluatorFactory.java      |   2 +-
 .../beam/runners/direct/TransformEvaluator.java |   2 +-
 .../direct/TransformEvaluatorRegistry.java      |   7 +
 .../beam/runners/direct/TransformExecutor.java  |   4 +-
 .../beam/runners/direct/TransformResult.java    |  16 +-
 .../direct/UnboundedReadEvaluatorFactory.java   |   3 +-
 .../runners/direct/ViewEvaluatorFactory.java    |   2 +-
 .../direct/WatermarkCallbackExecutor.java       |  34 ++
 .../beam/runners/direct/WatermarkManager.java   |  50 +-
 .../runners/direct/WindowEvaluatorFactory.java  |   6 +-
 .../direct/BoundedReadEvaluatorFactoryTest.java |  16 +-
 .../ConsumerTrackingPipelineVisitorTest.java    | 287 ---------
 .../runners/direct/DirectGraphVisitorTest.java  | 239 ++++++++
 ...leManagerRemovingTransformEvaluatorTest.java |   4 +-
 .../runners/direct/EvaluationContextTest.java   |  49 +-
 .../direct/FlattenEvaluatorFactoryTest.java     |   6 +-
 .../ImmutabilityCheckingBundleFactoryTest.java  |   6 +-
 .../ImmutabilityEnforcementFactoryTest.java     |   6 +-
 .../beam/runners/direct/ParDoEvaluatorTest.java |   2 +-
 .../beam/runners/direct/SplittableDoFnTest.java | 231 --------
 .../StatefulParDoEvaluatorFactoryTest.java      | 300 ++++++++++
 .../runners/direct/StepTransformResultTest.java |  25 +-
 .../direct/TestStreamEvaluatorFactoryTest.java  |  10 +-
 .../runners/direct/TransformExecutorTest.java   | 223 +++----
 .../UnboundedReadEvaluatorFactoryTest.java      |  20 +-
 .../runners/direct/WatermarkManagerTest.java    |  23 +-
 .../direct/WindowEvaluatorFactoryTest.java      |  16 +-
 runners/flink/runner/pom.xml                    |   9 +-
 .../apache/beam/runners/flink/FlinkRunner.java  |  12 +-
 .../FlinkBatchPipelineTranslator.java           |  14 +-
 .../FlinkStreamingPipelineTranslator.java       |  16 +-
 .../FlinkStreamingTransformTranslators.java     |  63 +-
 .../PipelineTranslationOptimizer.java           |  10 +-
 .../FlinkMergingNonShuffleReduceFunction.java   |   2 +-
 .../FlinkMergingPartialReduceFunction.java      |   2 +-
 .../functions/FlinkMergingReduceFunction.java   |   2 +-
 .../functions/FlinkPartialReduceFunction.java   |   2 +-
 .../functions/FlinkProcessContextBase.java      |   8 -
 .../functions/FlinkReduceFunction.java          |   2 +-
 runners/google-cloud-dataflow-java/pom.xml      |   4 +
 .../dataflow/DataflowPipelineTranslator.java    |  33 +-
 .../beam/runners/dataflow/DataflowRunner.java   |  53 +-
 .../DataflowPipelineTranslatorTest.java         |   2 +-
 .../runners/dataflow/DataflowRunnerTest.java    |   4 +-
 .../dataflow/RecordingPipelineVisitor.java      |   6 +-
 runners/spark/pom.xml                           |   7 +-
 .../runners/spark/SparkPipelineOptions.java     |   5 +
 .../apache/beam/runners/spark/SparkRunner.java  |  80 ++-
 .../spark/aggregators/AccumulatorSingleton.java |   2 +-
 .../beam/runners/spark/io/MicrobatchSource.java |  20 +-
 .../beam/runners/spark/io/SourceDStream.java    |   3 +-
 .../spark/stateful/StateSpecFunctions.java      |   2 +-
 .../spark/translation/BoundedDataset.java       |   5 +-
 .../beam/runners/spark/translation/Dataset.java |   2 +-
 .../spark/translation/EvaluationContext.java    |  10 +-
 .../spark/translation/SparkProcessContext.java  |   9 -
 .../translation/StorageLevelPTransform.java     |  43 ++
 .../spark/translation/TransformTranslator.java  |  27 +
 .../SparkRunnerStreamingContextFactory.java     |   2 +-
 .../streaming/StreamingTransformTranslator.java |   2 +-
 .../translation/streaming/UnboundedDataset.java |  13 +-
 .../spark/translation/StorageLevelTest.java     |  56 ++
 .../streaming/EmptyStreamAssertionTest.java     |   2 +
 .../streaming/FlattenStreamingTest.java         |   2 +
 .../streaming/SimpleStreamingWordCountTest.java |   1 +
 .../SparkTestPipelineOptionsForStreaming.java   |   6 -
 sdks/java/core/pom.xml                          |  29 +-
 .../beam/sdk/AggregatorPipelineExtractor.java   |   6 +-
 .../main/java/org/apache/beam/sdk/Pipeline.java | 151 ++---
 .../org/apache/beam/sdk/io/FileBasedSink.java   |  22 +-
 .../java/org/apache/beam/sdk/io/PubsubIO.java   | 176 +++++-
 .../apache/beam/sdk/io/PubsubUnboundedSink.java |  23 +-
 .../beam/sdk/io/PubsubUnboundedSource.java      |  40 +-
 .../java/org/apache/beam/sdk/io/TextIO.java     |  28 +-
 .../java/org/apache/beam/sdk/io/XmlSink.java    |   4 +-
 .../beam/sdk/options/PipelineOptions.java       |   7 +
 .../sdk/options/PipelineOptionsFactory.java     |   1 +
 .../sdk/options/ProxyInvocationHandler.java     |  26 +
 .../apache/beam/sdk/options/ValueProvider.java  |   2 +-
 .../beam/sdk/runners/TransformHierarchy.java    | 340 +++++++++--
 .../beam/sdk/runners/TransformTreeNode.java     | 257 --------
 .../beam/sdk/testing/FileChecksumMatcher.java   | 168 +++++-
 .../apache/beam/sdk/testing/GatherAllPanes.java |  88 +++
 .../org/apache/beam/sdk/testing/PAssert.java    |  77 +--
 .../apache/beam/sdk/testing/PaneExtractors.java |  55 +-
 .../beam/sdk/testing/UsesSplittableParDo.java   |  25 +
 .../beam/sdk/testing/ValueInSingleWindow.java   | 134 +++++
 .../org/apache/beam/sdk/transforms/Combine.java |  37 +-
 .../org/apache/beam/sdk/transforms/DoFn.java    |  16 +-
 .../beam/sdk/transforms/DoFnAdapters.java       |   2 +
 .../apache/beam/sdk/transforms/DoFnTester.java  | 350 ++++++-----
 .../org/apache/beam/sdk/transforms/OldDoFn.java |   8 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |  10 -
 .../reflect/ByteBuddyDoFnInvokerFactory.java    |  28 +-
 .../reflect/ByteBuddyOnTimerInvokerFactory.java |  10 +-
 .../sdk/transforms/reflect/DoFnInvoker.java     |   3 +
 .../sdk/transforms/reflect/DoFnInvokers.java    |  16 +-
 .../sdk/transforms/reflect/DoFnSignature.java   |   8 +
 .../apache/beam/sdk/util/GatherAllPanes.java    |  86 ---
 .../apache/beam/sdk/util/IdentityWindowFn.java  |   2 +-
 .../beam/sdk/util/PerKeyCombineFnRunners.java   | 258 --------
 .../apache/beam/sdk/util/TimerInternals.java    | 107 ++--
 .../beam/sdk/util/WindowingInternals.java       |  10 -
 .../beam/sdk/util/common/ReflectHelpers.java    |   3 +-
 .../sdk/util/state/InMemoryTimerInternals.java  |   8 +-
 .../sdk/util/state/TimerInternalsFactory.java   |  36 ++
 .../sdk/AggregatorPipelineExtractorTest.java    |  20 +-
 .../org/apache/beam/sdk/WindowMatchers.java     | 204 -------
 .../org/apache/beam/sdk/WindowMatchersTest.java |  82 ---
 .../org/apache/beam/sdk/io/PubsubIOTest.java    |  43 +-
 .../beam/sdk/io/PubsubUnboundedSinkTest.java    |  20 +-
 .../beam/sdk/io/PubsubUnboundedSourceTest.java  |  14 +-
 .../org/apache/beam/sdk/io/XmlSinkTest.java     |   6 +-
 .../beam/sdk/options/PipelineOptionsTest.java   |  24 +
 .../beam/sdk/options/ValueProviderTest.java     |  18 +
 .../sdk/runners/TransformHierarchyTest.java     | 230 ++++++++
 .../beam/sdk/runners/TransformTreeTest.java     |  12 +-
 .../sdk/testing/FileChecksumMatcherTest.java    | 131 +++-
 .../beam/sdk/testing/GatherAllPanesTest.java    | 140 +++++
 .../beam/sdk/testing/PaneExtractorsTest.java    | 133 ++---
 .../testing/ValueInSingleWindowCoderTest.java   |  51 ++
 .../beam/sdk/transforms/DoFnTesterTest.java     |  38 +-
 .../apache/beam/sdk/transforms/ParDoTest.java   | 177 +++++-
 .../beam/sdk/transforms/SplittableDoFnTest.java | 401 +++++++++++++
 .../display/DisplayDataEvaluator.java           |   8 +-
 .../beam/sdk/util/GatherAllPanesTest.java       | 143 -----
 .../apache/beam/sdk/util/ReleaseInfoTest.java   |  45 ++
 .../beam/sdk/util/TimerInternalsTest.java       |   4 +-
 .../sorter/BufferedExternalSorter.java          |   6 +-
 .../sdk/extensions/sorter/ExternalSorter.java   |   6 +-
 .../sorter/BufferedExternalSorterTest.java      |  58 +-
 .../extensions/sorter/ExternalSorterTest.java   |  53 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java     |  52 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java |  26 +-
 .../apache/beam/sdk/io/mongodb/MongoDbIO.java   |   6 +-
 .../examples-java8/generate-sources.sh          |  82 +++
 .../maven-archetypes/examples-java8/pom.xml     | 177 ++++++
 .../META-INF/maven/archetype-metadata.xml       |  39 ++
 .../main/resources/archetype-resources/pom.xml  | 209 +++++++
 .../projects/basic/archetype.properties         |  21 +
 .../src/test/resources/projects/basic/goal.txt  |   1 +
 .../examples/generate-sources.sh                |  68 +++
 sdks/java/maven-archetypes/examples/pom.xml     |  27 +-
 .../main/resources/archetype-resources/pom.xml  | 123 +++-
 .../src/main/java/DebuggingWordCount.java       | 164 ------
 .../src/main/java/MinimalWordCount.java         | 118 ----
 .../src/main/java/WindowedWordCount.java        | 229 -------
 .../src/main/java/WordCount.java                | 186 ------
 .../common/ExampleBigQueryTableOptions.java     |  55 --
 .../src/main/java/common/ExampleOptions.java    |  37 --
 ...xamplePubsubTopicAndSubscriptionOptions.java |  45 --
 .../java/common/ExamplePubsubTopicOptions.java  |  45 --
 .../src/main/java/common/ExampleUtils.java      | 352 -----------
 .../src/test/java/DebuggingWordCountTest.java   |  52 --
 .../src/test/java/WordCountTest.java            |  85 ---
 sdks/java/maven-archetypes/pom.xml              |  38 +-
 .../main/resources/archetype-resources/pom.xml  |   8 +
 .../resources/projects/basic/reference/pom.xml  |   8 +
 .../update-examples-archetype.sh                |  59 --
 229 files changed, 7944 insertions(+), 5261 deletions(-)
----------------------------------------------------------------------



Mime
View raw message