nemo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeongy...@apache.org
Subject [incubator-nemo] 01/02: merge master
Date Thu, 08 Mar 2018 23:02:49 GMT
This is an automated email from the ASF dual-hosted git repository.

jeongyoon pushed a commit to branch 717-TGE
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git

commit 653bda4caa933f19183f51eff821e6353d4db5c6
Merge: f48f2a4 7d8b9c3
Author: Jeongyoon Eo <jeongyoon0807@gmail.com>
AuthorDate: Thu Mar 8 19:24:27 2018 +0900

    merge master

 .travis.yml                                        |   7 +-
 README.md                                          |  11 +-
 .../main/java/edu/snu/nemo/common/ir/Readable.java |  10 +
 .../DuplicateEdgeGroupPropertyValue.java           |   1 +
 .../common/ir/vertex/InitializedSourceVertex.java  |   7 +-
 .../edu/snu/nemo/common/test/ExampleTestUtil.java  |   5 +
 compiler/frontend/beam/pom.xml                     |  11 +
 .../beam/source/BeamBoundedSourceVertex.java       |  19 +-
 .../spark/source/SparkBoundedSourceVertex.java     |   5 +
 .../optimizer/examples/EmptyComponents.java        |   4 +
 conf/src/main/java/edu/snu/nemo/conf/JobConf.java  |   2 +-
 .../beam/AlternatingLeastSquareITCase.java         |  25 +-
 .../snu/nemo/examples/beam/BroadcastITCase.java    |  17 +-
 .../snu/nemo/examples/beam/MapReduceITCase.java    |  22 +-
 .../beam/MultinomialLogisticRegressionITCase.java  |  40 +-
 examples/resources/sample_executor_resources.json  |   6 +-
 .../common/message/ncs/NcsMessageEnvironment.java  |   1 -
 .../common/plan/physical/BoundedSourceTask.java    |   3 +-
 runtime/common/src/main/proto/ControlMessage.proto |   3 +-
 .../edu/snu/nemo/driver/UserApplicationRunner.java |  17 +-
 .../runtime/executor/data/LimitedInputStream.java  |   1 -
 .../executor/data/streamchainer/Serializer.java    |   2 +-
 .../nemo/runtime/master/BlockManagerMaster.java    |  55 ++-
 .../edu/snu/nemo/runtime/master/RuntimeMaster.java | 175 +++-----
 .../runtime/master/resource/ContainerManager.java  | 198 +++------
 .../master/resource/ExecutorRepresenter.java       |  74 +++-
 .../master/scheduler/BatchSingleJobScheduler.java  | 311 ++++++-------
 .../runtime/master/scheduler/ExecutorRegistry.java | 204 +++++++++
 .../scheduler/RoundRobinSchedulingPolicy.java      | 193 ++++----
 .../nemo/runtime/master/scheduler/Scheduler.java   |   9 +-
 .../runtime/master/scheduler/SchedulerRunner.java  |  13 +-
 .../runtime/master/scheduler/SchedulingPolicy.java |  16 +-
 .../SourceLocationAwareSchedulingPolicy.java       | 242 ++++++++++
 .../snu/nemo/tests/compiler/CompilerTestUtil.java  |   2 +-
 .../snu/nemo/tests/runtime/RuntimeTestUtil.java    | 183 ++------
 .../runtime/executor/TaskGroupExecutorTest.java    |   4 +
 .../executor/datatransfer/DataTransferTest.java    |  18 +-
 .../tests/runtime/master/ContainerManagerTest.java |  42 +-
 .../scheduler/BatchSingleJobSchedulerTest.java     | 176 ++++----
 .../master/scheduler/FaultToleranceTest.java       | 154 +++----
 .../scheduler/RoundRobinSchedulingPolicyTest.java  |  81 ++--
 .../SourceLocationAwareSchedulingPolicyTest.java   | 492 +++++++++++++++++++++
 42 files changed, 1878 insertions(+), 983 deletions(-)

diff --cc examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
index 2d96b6e,7eac5f5..9fbc319
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/AlternatingLeastSquareITCase.java
@@@ -77,7 -70,9 +70,9 @@@ public final class AlternatingLeastSqua
    public void testPado() throws Exception {
      JobLauncher.main(builder
          .addJobId(AlternatingLeastSquareITCase.class.getSimpleName() + "_pado")
+         .addUserMain(AlternatingLeastSquare.class.getCanonicalName())
+         .addUserArgs(input, numFeatures, numIteration, lambda, output)
 -        .addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
 +        .addOptimizationPolicy(PadoPolicyParallelismFive.class.getCanonicalName())
          .build());
    }
  }
diff --cc examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
index b918960,ec1dbc5..d78431a
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/BroadcastITCase.java
@@@ -71,7 -68,9 +68,9 @@@ public final class BroadcastITCase 
    public void testPado() throws Exception {
      JobLauncher.main(builder
          .addJobId(BroadcastITCase.class.getSimpleName() + "_pado")
+         .addUserMain(Broadcast.class.getCanonicalName())
+         .addUserArgs(inputFilePath, outputFilePath)
 -        .addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
 +        .addOptimizationPolicy(PadoPolicyParallelismFive.class.getCanonicalName())
          .build());
    }
  }
diff --cc examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
index 1f96af5,9ff5100..72b1035
--- a/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
+++ b/examples/beam/src/test/java/edu/snu/nemo/examples/beam/MapReduceITCase.java
@@@ -70,7 -68,9 +68,9 @@@ public final class MapReduceITCase 
    public void testSailfish() throws Exception {
      JobLauncher.main(builder
          .addJobId(MapReduceITCase.class.getSimpleName() + "_sailfish")
+         .addUserMain(MapReduce.class.getCanonicalName())
+         .addUserArgs(inputFilePath, outputFilePath)
 -        .addOptimizationPolicy(SailfishPolicyParallelsimFive.class.getCanonicalName())
 +        .addOptimizationPolicy(SailfishPolicyParallelismFive.class.getCanonicalName())
          .build());
    }
  
@@@ -78,7 -78,9 +78,9 @@@
    public void testDisagg() throws Exception {
      JobLauncher.main(builder
          .addJobId(MapReduceITCase.class.getSimpleName() + "_disagg")
+         .addUserMain(MapReduce.class.getCanonicalName())
+         .addUserArgs(inputFilePath, outputFilePath)
 -        .addOptimizationPolicy(DisaggregationPolicyParallelsimFive.class.getCanonicalName())
 +        .addOptimizationPolicy(DisaggregationPolicyParallelismFive.class.getCanonicalName())
          .build());
    }
  
@@@ -86,7 -88,9 +88,9 @@@
    public void testPado() throws Exception {
      JobLauncher.main(builder
          .addJobId(MapReduceITCase.class.getSimpleName() + "_pado")
+         .addUserMain(MapReduce.class.getCanonicalName())
+         .addUserArgs(inputFilePath, outputFilePath)
 -        .addOptimizationPolicy(PadoPolicyParallelsimFive.class.getCanonicalName())
 +        .addOptimizationPolicy(PadoPolicyParallelismFive.class.getCanonicalName())
          .build());
    }
  
@@@ -98,7 -102,9 +102,9 @@@
    public void testDataSkew() throws Exception {
      JobLauncher.main(builder
          .addJobId(MapReduceITCase.class.getSimpleName() + "_dataskew")
+         .addUserMain(MapReduce.class.getCanonicalName())
+         .addUserArgs(inputFilePath, outputFilePath)
 -        .addOptimizationPolicy(DataSkewPolicyParallelsimFive.class.getCanonicalName())
 +        .addOptimizationPolicy(DataSkewPolicyParallelismFive.class.getCanonicalName())
          .build());
    }
  }
diff --cc runtime/master/src/main/java/edu/snu/nemo/runtime/master/BlockManagerMaster.java
index 0651b9b,4dc2e4c..acc4dfd
--- a/runtime/master/src/main/java/edu/snu/nemo/runtime/master/BlockManagerMaster.java
+++ b/runtime/master/src/main/java/edu/snu/nemo/runtime/master/BlockManagerMaster.java
@@@ -16,12 -16,13 +16,11 @@@
  package edu.snu.nemo.runtime.master;
  
  import edu.snu.nemo.common.exception.IllegalMessageException;
- import edu.snu.nemo.runtime.common.comm.ControlMessage;
+ import edu.snu.nemo.common.exception.UnknownExecutionStateException;
  import edu.snu.nemo.runtime.common.exception.AbsentBlockException;
  import edu.snu.nemo.runtime.common.RuntimeIdGenerator;
 -import edu.snu.nemo.runtime.common.comm.ControlMessage;
  import edu.snu.nemo.runtime.common.message.MessageContext;
  import edu.snu.nemo.runtime.common.message.MessageEnvironment;
--import edu.snu.nemo.runtime.common.message.MessageListener;
  import edu.snu.nemo.runtime.common.state.BlockState;
  
  import com.google.common.annotations.VisibleForTesting;

-- 
To stop receiving notification emails like this one, please contact
jeongyoon@apache.org.

Mime
View raw message