nemo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jooy...@apache.org
Subject [incubator-nemo] 02/02: Merge branch 'master' into maven-artifact
Date Thu, 01 Nov 2018 01:54:06 GMT
This is an automated email from the ASF dual-hosted git repository.

jooykim pushed a commit to branch maven-artifact
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git

commit 49266b5b70aa7bc317807d20e2dd7a1cfb170ffd
Merge: b2787ca eea3f57
Author: Joo Yeon Kim <jooykim@apache.org>
AuthorDate: Wed Oct 31 11:05:38 2018 +0900

    Merge branch 'master' into maven-artifact

 .gitignore                                         |   1 +
 .travis.yml                                        |   4 +-
 checkstyle.xml                                     |   2 +-
 .../java/org/apache/nemo/client/JobLauncher.java   |  15 +-
 ...Collector.java => BoundedIteratorReadable.java} |  43 ++--
 .../org/apache/nemo/common/ir/OutputCollector.java |   7 +
 .../java/org/apache/nemo/common/ir/Readable.java   |  38 +++-
 .../edge/executionproperty/DataStoreProperty.java  |   1 +
 .../nemo/common/ir/vertex/CachedSourceVertex.java  |  40 +++-
 .../common/ir/vertex/InMemorySourceVertex.java     |  25 ++-
 .../apache/nemo/common/ir/vertex/SourceVertex.java |   2 +
 .../vertex/transform/AggregateMetricTransform.java |   2 +-
 .../vertex/transform/MetricCollectTransform.java   |   2 +-
 .../vertex/transform/NoWatermarkEmitTransform.java |  24 +--
 .../common/ir/vertex/transform/RelayTransform.java |   6 +
 .../nemo/common/ir/vertex/transform/Transform.java |  10 +
 .../apache/nemo/common/punctuation/Finishmark.java |  23 +--
 .../Watermark.java}                                |  48 +++--
 .../org/apache/nemo/common/test/ArgBuilder.java    |   9 +
 .../apache/nemo/common/test/EmptyComponents.java   |  37 +++-
 .../compiler/frontend/beam/PipelineTranslator.java |  21 +-
 .../beam/source/BeamBoundedSourceVertex.java       |  73 +++++--
 .../beam/source/BeamUnboundedSourceVertex.java     | 128 ++++++------
 .../beam/transform/AbstractDoFnTransform.java      |   6 +-
 .../beam/transform/CreateViewTransform.java        |   4 +-
 .../frontend/beam/transform/DoFnTransform.java     |   8 +
 .../frontend/beam/transform/FlattenTransform.java  |   6 +
 .../GroupByKeyAndWindowDoFnTransform.java          |  10 +-
 .../beam/transform/GroupByKeyTransform.java        |   4 +-
 .../frontend/beam/transform/WindowFnTransform.java |   6 +
 compiler/frontend/spark/pom.xml                    |  19 ++
 .../source/SparkDatasetBoundedSourceVertex.java    |  29 ++-
 .../source/SparkTextFileBoundedSourceVertex.java   |  34 +++-
 .../frontend/spark/transform/CollectTransform.java |   5 +-
 .../frontend/spark/transform/FlatMapTransform.java |   6 +
 .../spark/transform/GroupByKeyTransform.java       |   3 +-
 .../spark/transform/HDFSTextFileTransform.java     |   3 +-
 .../spark/transform/LocalTextFileTransform.java    |   4 +-
 .../spark/transform/MapToPairTransform.java        |   6 +
 .../frontend/spark/transform/MapTransform.java     |   6 +
 .../spark/transform/ReduceByKeyTransform.java      |   4 +-
 .../frontend/spark/transform/ReduceTransform.java  |   6 +
 .../annotating/PipeTransferForAllEdgesPass.java    |  41 ++--
 .../frontend/beam/transform/DoFnTransformTest.java |   6 +
 .../main/java/org/apache/nemo/conf/JobConf.java    |   8 +
 .../nemo/examples/beam/WriteOneFilePerWindow.java  |   2 +-
 .../examples/beam/WindowedWordCountITCase.java     |  48 ++++-
 .../policy/StreamingPolicyParallelismFive.java     |  57 ++++++
 pom.xml                                            |   9 +
 .../runtime/common/message/MessageEnvironment.java |   1 +
 .../common/message/ncs/NcsMessageEnvironment.java  |   7 +
 .../nemo/runtime/common/plan/RuntimeEdge.java      |  15 ++
 runtime/common/src/main/proto/ControlMessage.proto |  33 +++-
 .../org/apache/nemo/runtime/executor/Executor.java |  10 +-
 .../executor/bytetransfer/ByteTransfer.java        |  12 +-
 .../executor/bytetransfer/ByteTransferContext.java |  10 +-
 .../ByteTransportChannelInitializer.java           |  11 +-
 .../bytetransfer/ClosableBlockingQueue.java        |   1 +
 .../executor/bytetransfer/ContextManager.java      |  40 +++-
 .../executor/bytetransfer/ControlFrameEncoder.java |   1 +
 .../runtime/executor/data/BlockManagerWorker.java  |  10 +-
 .../executor/data/BroadcastManagerWorker.java      |   3 +-
 .../nemo/runtime/executor/data/PipeContainer.java  | 150 ++++++++++++++
 .../runtime/executor/data/PipeManagerWorker.java   | 199 +++++++++++++++++++
 .../runtime/executor/data/SerializerManager.java   |   2 +-
 .../data/partitioner/DataSkewHashPartitioner.java  |   3 -
 .../executor/data/stores/GlusterFileStore.java     |   3 -
 .../executor/data/stores/LocalFileStore.java       |   3 -
 .../{InputReader.java => BlockInputReader.java}    | 125 ++++--------
 .../{OutputWriter.java => BlockOutputWriter.java}  |  90 +++------
 .../datatransfer/DataFetcherOutputCollector.java   |   6 +
 .../datatransfer/DynOptDataOutputCollector.java    |   6 +
 .../runtime/executor/datatransfer/InputReader.java | 155 +--------------
 ...Factory.java => IntermediateDataIOFactory.java} |  35 +++-
 .../OperatorVertexOutputCollector.java             |  20 ++
 .../executor/datatransfer/OutputWriter.java        | 158 ++++-----------
 .../executor/datatransfer/PipeInputReader.java     |  82 ++++++++
 .../executor/datatransfer/PipeOutputWriter.java    | 136 +++++++++++++
 .../nemo/runtime/executor/task/DataFetcher.java    |   5 +-
 .../executor/task/ParentTaskDataFetcher.java       |  12 +-
 .../executor/task/SourceVertexDataFetcher.java     |  81 ++++++--
 .../nemo/runtime/executor/task/TaskExecutor.java   | 203 ++++++++++++++-----
 .../executor/datatransfer/DataTransferTest.java    |  77 +++++---
 .../executor/task/ParentTaskDataFetcherTest.java   |  13 +-
 .../runtime/executor/task/TaskExecutorTest.java    | 219 +++++++++++++++++++--
 .../apache/nemo/runtime/master/MetricStore.java    |   2 +
 .../nemo/runtime/master/PipeManagerMaster.java     | 157 +++++++++++++++
 .../apache/nemo/runtime/master/RuntimeMaster.java  |   1 +
 .../master/resource/ExecutorRepresenter.java       |   6 +-
 .../master/resource/ResourceSpecification.java     |   2 +-
 .../nemo/runtime/master/scheduler/Scheduler.java   |   2 -
 .../master/scheduler/StreamingScheduler.java       |  29 ++-
 .../runtime/master/scheduler/TaskDispatcher.java   |  10 +-
 .../master/scheduler/StreamingSchedulerTest.java   |   8 +-
 .../runtime/master/scheduler/TaskRetryTest.java    |   2 +-
 95 files changed, 2214 insertions(+), 843 deletions(-)



Mime
View raw message