beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amits...@apache.org
Subject [23/23] beam git commit: This closes #2050
Date Tue, 28 Feb 2017 22:35:30 GMT
This closes #2050


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/61e31e62
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/61e31e62
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/61e31e62

Branch: refs/heads/master
Commit: 61e31e622ec5b1f3c4cd1417c859810689f2683c
Parents: a41afdc 2c70743
Author: Sela <ansela@paypal.com>
Authored: Wed Mar 1 00:18:45 2017 +0200
Committer: Sela <ansela@paypal.com>
Committed: Wed Mar 1 00:18:45 2017 +0200

----------------------------------------------------------------------
 .../apache/beam/runners/core/LateDataUtils.java |  88 ++++
 .../core/UnsupportedSideInputReader.java        |  52 +++
 runners/spark/pom.xml                           |   5 +-
 .../runners/spark/SparkPipelineOptions.java     |   6 +-
 .../beam/runners/spark/SparkPipelineResult.java |   4 +
 .../apache/beam/runners/spark/SparkRunner.java  |   8 +-
 .../runners/spark/TestSparkPipelineOptions.java |  35 ++
 .../beam/runners/spark/TestSparkRunner.java     |  88 ++--
 .../aggregators/AggregatorsAccumulator.java     |   2 +-
 .../beam/runners/spark/coders/CoderHelpers.java |  23 +
 .../apache/beam/runners/spark/io/ConsoleIO.java |   4 +-
 .../beam/runners/spark/io/CreateStream.java     | 198 +++++++--
 .../spark/metrics/MetricsAccumulator.java       |   4 -
 .../spark/metrics/SparkMetricsContainer.java    |   5 +-
 .../SparkGroupAlsoByWindowViaWindowSet.java     | 431 +++++++++++++++++++
 .../spark/stateful/SparkStateInternals.java     | 418 ++++++++++++++++++
 .../spark/stateful/SparkTimerInternals.java     | 185 ++++++++
 .../spark/translation/EvaluationContext.java    |   6 -
 .../translation/GroupCombineFunctions.java      | 237 ++++------
 .../translation/SparkGroupAlsoByWindowFn.java   | 198 ---------
 ...SparkGroupAlsoByWindowViaOutputBufferFn.java | 179 ++++++++
 .../spark/translation/TransformTranslator.java  | 238 ++++++----
 .../spark/translation/TranslationUtils.java     |  24 +-
 .../streaming/StreamingTransformTranslator.java | 277 +++++-------
 .../translation/streaming/UnboundedDataset.java |  59 +--
 .../beam/runners/spark/ForceStreamingTest.java  |  17 +-
 .../spark/GlobalWatermarkHolderTest.java        | 151 +++++++
 .../apache/beam/runners/spark/PipelineRule.java | 120 ++++++
 .../runners/spark/ProvidedSparkContextTest.java |   2 +-
 .../beam/runners/spark/ReuseSparkContext.java   |  46 --
 .../runners/spark/ReuseSparkContextRule.java    |  46 ++
 .../runners/spark/SparkPipelineStateTest.java   |  33 +-
 .../beam/runners/spark/WatermarkTest.java       | 231 ----------
 .../metrics/sink/NamedAggregatorsTest.java      |  11 +-
 .../beam/runners/spark/io/AvroPipelineTest.java |   6 +-
 .../beam/runners/spark/io/NumShardsTest.java    |   6 +-
 .../io/hadoop/HadoopFileFormatPipelineTest.java |   6 +-
 .../spark/translation/StorageLevelTest.java     |   8 +-
 .../translation/streaming/CreateStreamTest.java | 369 ++++++++++++++++
 .../streaming/EmptyStreamAssertionTest.java     |  87 ----
 .../streaming/FlattenStreamingTest.java         |  82 ----
 .../streaming/KafkaStreamingTest.java           | 270 ------------
 .../ResumeFromCheckpointStreamingTest.java      | 292 +++++++++----
 .../streaming/SimpleStreamingWordCountTest.java |  84 ----
 .../streaming/TrackStreamingSourcesTest.java    |  45 +-
 .../utils/KafkaWriteOnBatchCompleted.java       | 105 -----
 .../streaming/utils/PAssertStreaming.java       | 121 ------
 .../utils/SparkTestPipelineOptions.java         |  42 --
 .../SparkTestPipelineOptionsForStreaming.java   |  37 --
 .../windowing/TimestampTransform.java           |   3 +-
 50 files changed, 2971 insertions(+), 2023 deletions(-)
----------------------------------------------------------------------



Mime
View raw message