beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t..@apache.org
Subject [49/50] incubator-beam git commit: Merge branch 'master' into apex-runner
Date Mon, 07 Nov 2016 19:59:39 GMT
Merge branch 'master' into apex-runner


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5ccd5fb3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5ccd5fb3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5ccd5fb3

Branch: refs/heads/apex-runner
Commit: 5ccd5fb3c62ebdf1310af4af17cab2270b4a0d96
Parents: 15287b8 1102455
Author: Thomas Weise <thw@apache.org>
Authored: Mon Nov 7 20:12:36 2016 +0100
Committer: Thomas Weise <thw@apache.org>
Committed: Mon Nov 7 20:12:36 2016 +0100

----------------------------------------------------------------------
 .travis.yml                                     |   6 +-
 examples/java/pom.xml                           |  79 --
 .../beam/examples/common/ExampleUtils.java      |   3 +-
 .../apache/beam/examples/complete/TfIdf.java    |   6 +-
 .../beam/examples/cookbook/DeDupExample.java    |   8 +-
 .../org/apache/beam/examples/cookbook/README.md |   2 +-
 .../beam/examples/WindowedWordCountIT.java      |  11 +-
 .../org/apache/beam/examples/WordCountIT.java   |  19 +-
 .../beam/examples/complete/TfIdfTest.java       |   4 +-
 .../examples/cookbook/BigQueryTornadoesIT.java  |  11 +-
 .../examples/cookbook/DeDupExampleTest.java     |  82 --
 .../examples/cookbook/DistinctExampleTest.java  |  82 ++
 examples/java8/pom.xml                          |  19 -
 .../beam/examples/complete/game/GameStats.java  |   3 +-
 .../examples/complete/game/LeaderBoard.java     |   2 +-
 .../beam/examples/complete/game/UserScore.java  |   2 +-
 .../examples/MinimalWordCountJava8Test.java     |   2 +-
 examples/pom.xml                                |  13 +-
 pom.xml                                         | 107 ++-
 runners/core-java/pom.xml                       |  65 --
 .../core/GroupAlsoByWindowViaWindowSetDoFn.java |   3 +-
 .../GroupAlsoByWindowsViaOutputBufferDoFn.java  |   3 +-
 .../core/GroupByKeyViaGroupByKeyOnly.java       |  13 +-
 .../core/PushbackSideInputDoFnRunner.java       |  28 +-
 .../beam/runners/core/ReduceFnTester.java       |   6 +-
 .../UnboundedReadFromBoundedSourceTest.java     |   4 +-
 runners/direct-java/pom.xml                     | 102 ---
 .../runners/direct/AggregatorContainer.java     |  20 +-
 ...ectGBKIntoKeyedWorkItemsOverrideFactory.java |  16 +-
 .../beam/runners/direct/DirectGroupByKey.java   |  42 +-
 .../direct/DirectGroupByKeyOverrideFactory.java |  25 +-
 .../runners/direct/DirectTimerInternals.java    |  13 +
 .../beam/runners/direct/EvaluationContext.java  |  18 +-
 .../direct/ExecutorServiceParallelExecutor.java |  41 +-
 .../GroupAlsoByWindowEvaluatorFactory.java      | 249 ++++--
 .../direct/GroupByKeyOnlyEvaluatorFactory.java  |  31 +-
 .../direct/PTransformOverrideFactory.java       |   8 +-
 .../beam/runners/direct/ParDoEvaluator.java     |  28 +-
 .../runners/direct/ParDoEvaluatorFactory.java   | 126 ++++
 .../direct/ParDoMultiEvaluatorFactory.java      | 107 ---
 .../direct/ParDoMultiEvaluatorHooks.java        |  55 ++
 .../runners/direct/ParDoOverrideFactory.java    |  14 +-
 .../direct/ParDoSingleEvaluatorFactory.java     | 110 ---
 .../direct/ParDoSingleEvaluatorHooks.java       |  58 ++
 .../direct/TestStreamEvaluatorFactory.java      |  14 +-
 .../direct/TransformEvaluatorRegistry.java      |  10 +-
 .../direct/UncommittedBundleOutputManager.java  |  50 --
 .../runners/direct/ViewEvaluatorFactory.java    |  19 +-
 .../direct/WatermarkCallbackExecutor.java       |   3 +-
 .../beam/runners/direct/WatermarkManager.java   |  79 +-
 .../direct/WriteWithShardingFactory.java        |  15 +-
 .../runners/direct/EvaluationContextTest.java   |  23 +-
 .../direct/GroupByKeyEvaluatorFactoryTest.java  |  29 +-
 .../GroupByKeyOnlyEvaluatorFactoryTest.java     |  31 +-
 .../beam/runners/direct/ParDoEvaluatorTest.java |  16 +-
 .../direct/ParDoMultiEvaluatorFactoryTest.java  | 439 -----------
 .../direct/ParDoMultiEvaluatorHooksTest.java    | 439 +++++++++++
 .../direct/ParDoSingleEvaluatorFactoryTest.java | 335 ---------
 .../direct/ParDoSingleEvaluatorHooksTest.java   | 335 +++++++++
 .../runners/direct/WatermarkManagerTest.java    | 102 +--
 .../direct/WriteWithShardingFactoryTest.java    |  26 +-
 runners/flink/examples/pom.xml                  |  17 -
 .../beam/runners/flink/examples/TFIDF.java      |   6 +-
 .../flink/examples/streaming/AutoComplete.java  |   8 +-
 .../flink/examples/streaming/JoinExamples.java  |  12 +-
 .../examples/streaming/KafkaIOExamples.java     |   2 +-
 .../examples/streaming/WindowedWordCount.java   |   8 +-
 runners/flink/runner/pom.xml                    |  17 -
 .../wrappers/streaming/DoFnOperator.java        |  10 +-
 .../wrappers/streaming/WindowDoFnOperator.java  |  13 +
 runners/google-cloud-dataflow-java/pom.xml      | 114 ---
 .../dataflow/BlockingDataflowRunner.java        | 170 -----
 .../dataflow/DataflowJobCancelledException.java |  39 -
 .../dataflow/DataflowJobExecutionException.java |  35 -
 .../dataflow/DataflowJobUpdatedException.java   |  51 --
 .../runners/dataflow/DataflowPipelineJob.java   |  44 +-
 .../dataflow/DataflowPipelineRegistrar.java     |  11 +-
 .../dataflow/DataflowPipelineTranslator.java    |   8 +-
 .../beam/runners/dataflow/DataflowRunner.java   |  49 +-
 .../runners/dataflow/internal/IsmFormat.java    |   5 +-
 .../dataflow/internal/ReadTranslator.java       |  12 +-
 .../BlockingDataflowPipelineOptions.java        |  28 -
 .../options/DataflowPipelineOptions.java        |   8 +
 .../testing/TestDataflowPipelineOptions.java    |   6 +-
 .../dataflow/testing/TestDataflowRunner.java    |   7 +-
 .../dataflow/util/DataflowTemplateJob.java      |  70 ++
 .../beam/runners/dataflow/util/DoFnInfo.java    |  18 +-
 .../runners/dataflow/util/MonitoringUtil.java   |   1 -
 .../runners/dataflow/util/RandomAccessData.java |   2 +-
 .../dataflow/BlockingDataflowRunnerTest.java    | 300 --------
 .../dataflow/DataflowPipelineJobTest.java       |  30 +-
 .../dataflow/DataflowPipelineRegistrarTest.java |   7 +-
 .../DataflowPipelineTranslatorTest.java         |  28 +
 .../runners/dataflow/DataflowRunnerTest.java    |  45 ++
 runners/pom.xml                                 |  11 +
 runners/spark/pom.xml                           |  12 -
 .../runners/spark/SparkPipelineOptions.java     |   3 +-
 .../apache/beam/runners/spark/io/SourceRDD.java |   4 +-
 .../translation/GroupCombineFunctions.java      |   5 +-
 .../ReifyTimestampsAndWindowsFunction.java      |  47 ++
 .../spark/translation/SparkProcessContext.java  |  46 +-
 .../SparkRunnerStreamingContextFactory.java     |  23 +-
 .../streaming/EmptyStreamAssertionTest.java     |   3 +-
 .../streaming/FlattenStreamingTest.java         |   6 +-
 .../streaming/KafkaStreamingTest.java           |  10 +-
 .../ResumeFromCheckpointStreamingTest.java      |   3 +-
 .../streaming/SimpleStreamingWordCountTest.java |   3 +-
 .../utils/TestOptionsForStreaming.java          |  12 +-
 sdks/java/build-tools/pom.xml                   |   4 +-
 .../src/main/resources/beam/checkstyle.xml      |   9 +-
 sdks/java/core/pom.xml                          | 133 +---
 .../java/org/apache/beam/sdk/io/AvroIO.java     |   4 +-
 .../sdk/io/BoundedReadFromUnboundedSource.java  |   4 +-
 .../org/apache/beam/sdk/io/BoundedSource.java   |  11 +-
 .../apache/beam/sdk/io/CompressedSource.java    |   4 +-
 .../org/apache/beam/sdk/io/FileBasedSink.java   |  43 +-
 .../org/apache/beam/sdk/io/FileBasedSource.java |  59 +-
 .../beam/sdk/io/PubsubUnboundedSource.java      | 123 ++-
 .../main/java/org/apache/beam/sdk/io/Sink.java  |   4 +-
 .../java/org/apache/beam/sdk/io/TextIO.java     |  43 +-
 .../java/org/apache/beam/sdk/io/XmlSource.java  |   2 +-
 .../apache/beam/sdk/io/range/ByteKeyRange.java  |   2 +-
 .../sdk/options/PipelineOptionsFactory.java     | 248 ++++--
 .../sdk/options/ProxyInvocationHandler.java     |   5 +-
 .../apache/beam/sdk/options/ValueProvider.java  |  38 +-
 .../apache/beam/sdk/transforms/Aggregator.java  |  13 +-
 .../apache/beam/sdk/transforms/CombineFns.java  |   4 +-
 .../apache/beam/sdk/transforms/Distinct.java    | 158 ++++
 .../org/apache/beam/sdk/transforms/DoFn.java    |  22 +-
 .../org/apache/beam/sdk/transforms/Flatten.java |   3 +-
 .../apache/beam/sdk/transforms/GroupByKey.java  |   4 +-
 .../org/apache/beam/sdk/transforms/ParDo.java   |   2 +-
 .../org/apache/beam/sdk/transforms/Regex.java   | 454 +++++++++++
 .../beam/sdk/transforms/RemoveDuplicates.java   | 159 ----
 .../org/apache/beam/sdk/transforms/View.java    |   2 +-
 .../sdk/transforms/display/DisplayData.java     |   5 +-
 .../beam/sdk/transforms/join/CoGroupByKey.java  |   6 +-
 .../sdk/transforms/reflect/DoFnInvokers.java    |   7 +-
 .../sdk/transforms/reflect/DoFnSignature.java   |  46 +-
 .../sdk/transforms/reflect/DoFnSignatures.java  | 752 ++++++++++++-------
 .../splittabledofn/RestrictionTracker.java      |   2 +-
 .../transforms/splittabledofn/package-info.java |   4 +-
 .../beam/sdk/transforms/windowing/Window.java   | 130 +---
 .../beam/sdk/util/BaseExecutionContext.java     |   2 +-
 .../BufferedElementCountingOutputStream.java    |   4 +-
 .../apache/beam/sdk/util/ExecutableTrigger.java | 131 ----
 .../apache/beam/sdk/util/ExecutionContext.java  |   8 +-
 .../sdk/util/ExposedByteArrayInputStream.java   |   5 +-
 .../sdk/util/ExposedByteArrayOutputStream.java  |   2 -
 .../beam/sdk/util/FileIOChannelFactory.java     |   8 +-
 .../apache/beam/sdk/util/GatherAllPanes.java    |  18 +-
 .../beam/sdk/util/GcsIOChannelFactory.java      |   8 +-
 .../java/org/apache/beam/sdk/util/GcsUtil.java  |   2 -
 .../apache/beam/sdk/util/IOChannelFactory.java  |   4 +
 .../apache/beam/sdk/util/MovingFunction.java    |   4 +-
 .../org/apache/beam/sdk/util/PubsubClient.java  |  32 +-
 .../apache/beam/sdk/util/PubsubTestClient.java  |  32 +
 .../sdk/util/ReifyTimestampAndWindowsDoFn.java  |  41 -
 .../sdk/util/ReifyTimestampsAndWindows.java     |  63 --
 .../org/apache/beam/sdk/util/StringUtils.java   |   8 +-
 .../apache/beam/sdk/util/TimerInternals.java    |  35 +-
 .../apache/beam/sdk/util/WindowingStrategy.java |  10 +-
 .../beam/sdk/util/common/ReflectHelpers.java    |  17 +
 .../org/apache/beam/sdk/util/gcsfs/GcsPath.java |  13 +-
 .../sdk/util/state/InMemoryTimerInternals.java  |  12 +
 .../beam/sdk/util/state/StateNamespace.java     |  16 +-
 .../java/org/apache/beam/sdk/values/PDone.java  |   2 +-
 .../apache/beam/sdk/values/TypeDescriptor.java  |  48 +-
 .../apache/beam/sdk/values/TypeDescriptors.java |   1 -
 .../apache/beam/sdk/values/TypeParameter.java   |  62 ++
 .../apache/beam/sdk/io/CountingInputTest.java   |   6 +-
 .../apache/beam/sdk/io/CountingSourceTest.java  |   8 +-
 .../apache/beam/sdk/io/FileBasedSinkTest.java   |  23 +-
 .../beam/sdk/io/PubsubUnboundedSourceTest.java  |  79 ++
 .../org/apache/beam/sdk/io/XmlSinkTest.java     |  16 +-
 .../sdk/options/PipelineOptionsFactoryTest.java | 239 ++++++
 .../sdk/options/ProxyInvocationHandlerTest.java |  29 +-
 .../beam/sdk/options/ValueProviderTest.java     |  33 +
 .../apache/beam/sdk/testing/PAssertTest.java    |  27 +
 .../sdk/transforms/ApproximateUniqueTest.java   |   2 +-
 .../beam/sdk/transforms/DistinctTest.java       | 130 ++++
 .../apache/beam/sdk/transforms/RegexTest.java   | 248 ++++++
 .../sdk/transforms/RemoveDuplicatesTest.java    | 130 ----
 .../DoFnSignaturesProcessElementTest.java       |  18 +-
 .../DoFnSignaturesSplittableDoFnTest.java       |  19 +-
 .../transforms/reflect/DoFnSignaturesTest.java  |  42 +-
 .../reflect/DoFnSignaturesTestUtils.java        |  13 +-
 .../sdk/transforms/windowing/WindowTest.java    |   6 +-
 .../beam/sdk/util/ExecutableTriggerTest.java    | 109 ---
 .../sdk/util/common/ReflectHelpersTest.java     |  25 +
 .../apache/beam/sdk/util/gcsfs/GcsPathTest.java |  25 +
 .../beam/sdk/values/TypeDescriptorTest.java     |  12 +
 sdks/java/extensions/join-library/pom.xml       |   8 -
 sdks/java/extensions/pom.xml                    |   1 +
 sdks/java/extensions/sorter/README.md           |  42 ++
 sdks/java/extensions/sorter/pom.xml             | 159 ++++
 .../sorter/BufferedExternalSorter.java          | 125 +++
 .../sdk/extensions/sorter/ExternalSorter.java   | 225 ++++++
 .../sdk/extensions/sorter/InMemorySorter.java   | 166 ++++
 .../beam/sdk/extensions/sorter/SortValues.java  | 213 ++++++
 .../beam/sdk/extensions/sorter/Sorter.java      |  47 ++
 .../sdk/extensions/sorter/package-info.java     |  23 +
 .../sorter/BufferedExternalSorterTest.java      | 177 +++++
 .../extensions/sorter/ExternalSorterTest.java   |  87 +++
 .../extensions/sorter/InMemorySorterTest.java   | 144 ++++
 .../sdk/extensions/sorter/SortValuesTest.java   | 128 ++++
 .../sdk/extensions/sorter/SorterTestUtils.java  | 129 ++++
 sdks/java/io/google-cloud-platform/pom.xml      |   8 -
 .../beam/sdk/io/gcp/datastore/DatastoreV1.java  |   2 +-
 .../apache/beam/sdk/io/gcp/ApiSurfaceTest.java  | 132 ++++
 sdks/java/io/hdfs/pom.xml                       |   8 -
 sdks/java/io/jdbc/pom.xml                       |  14 +-
 .../org/apache/beam/sdk/io/jdbc/JdbcIO.java     |  39 +
 .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java |  33 +
 sdks/java/io/jms/pom.xml                        |   8 -
 sdks/java/io/kafka/pom.xml                      |  15 -
 .../apache/beam/sdk/io/kafka/KafkaIOTest.java   |   6 +-
 sdks/java/io/kinesis/pom.xml                    |  15 -
 .../apache/beam/sdk/io/kinesis/KinesisIO.java   |  35 +-
 sdks/java/io/mongodb/pom.xml                    |  12 -
 sdks/java/java8tests/pom.xml                    |  11 -
 .../beam/sdk/transforms/DistinctJava8Test.java  | 104 +++
 .../transforms/RemoveDuplicatesJava8Test.java   | 104 ---
 .../main/resources/archetype-resources/pom.xml  |  32 -
 sdks/java/microbenchmarks/pom.xml               |  10 -
 sdks/java/pom.xml                               |   9 +-
 sdks/pom.xml                                    |  16 +-
 227 files changed, 7001 insertions(+), 4602 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5ccd5fb3/examples/java/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5ccd5fb3/runners/pom.xml
----------------------------------------------------------------------


Mime
View raw message