myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mo...@apache.org
Subject [23/50] [abbrv] incubator-myriad git commit: Address review comments; fix findBugs and build issues - Defined a new 'myriadSchedulerConf' to exclude hadoop dependencies from being copied into myriad-scheduler/build/libs. Tests are now unaffected. - Fixed
Date Fri, 25 Sep 2015 22:46:03 GMT
Address review comments; fix findBugs and build issues
- Defined a new 'myriadSchedulerConf' to exclude hadoop dependencies from being copied into
myriad-scheduler/build/libs. Tests are now unaffected.
- Fixed a few findBugs issues.
- In ResourceOffersEventHandler, fixed code to ensure an offer used to launch a zero profile
NM is not used again for FGS.
- Fixed a test in TestMyriadScheduler. The test for NodeStore needs to be coded differently
due to the recent FGS changes. Will be done in a separate changeset.


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/ef328d15
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/ef328d15
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/ef328d15

Branch: refs/heads/master
Commit: ef328d15ffc7fd378fe7ce8417b665d462cbfa72
Parents: 5611a8c
Author: Santosh Marella <smarella@maprtech.com>
Authored: Wed Aug 5 16:55:35 2015 -0700
Committer: Santosh Marella <smarella@maprtech.com>
Committed: Wed Aug 5 16:57:55 2015 -0700

----------------------------------------------------------------------
 build.gradle                                      |  3 ---
 myriad-scheduler/build.gradle                     |  9 +++++++--
 .../src/main/java/com/ebay/myriad/Main.java       |  9 +++++----
 .../myriad/policy/LeastAMNodesFirstPolicy.java    |  2 +-
 .../handlers/ResourceOffersEventHandler.java      |  5 ++++-
 .../scheduler/fgs/OfferLifecycleManager.java      |  1 -
 .../yarn/interceptor/CompositeInterceptor.java    |  8 ++++----
 .../interceptor/YarnSchedulerInterceptor.java     |  3 +++
 .../myriad/scheduler/TestMyriadScheduler.java     | 18 +++---------------
 9 files changed, 27 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 25fa361..4136792 100644
--- a/build.gradle
+++ b/build.gradle
@@ -45,9 +45,6 @@ subprojects {
         capsule
         myriadExecutorConf
         myriadExecutorConf.transitive = false
-        // exclude hadoop/yarn deps for 'runtime'
-        // TODO (Kannan Rajah) Exclude only while packaging
-        //runtime.exclude group: 'org.apache.hadoop', module: '*'
     }
 
     repositories {

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/myriad-scheduler/build.gradle
----------------------------------------------------------------------
diff --git a/myriad-scheduler/build.gradle b/myriad-scheduler/build.gradle
index bdb0cfe..b3ee397 100644
--- a/myriad-scheduler/build.gradle
+++ b/myriad-scheduler/build.gradle
@@ -14,12 +14,17 @@ dependencies {
 	compile "com.fasterxml.jackson.core:jackson-databind:2.5.1"
 	compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.5.1"
 	compile "org.apache.curator:curator-framework:2.7.1"
-  testCompile "org.apache.hadoop:hadoop-yarn-server-resourcemanager:${hadoopVer}:tests"
+    testCompile "org.apache.hadoop:hadoop-yarn-server-resourcemanager:${hadoopVer}:tests"
+}
+
+configurations {
+    myriadSchedulerConf.extendsFrom(runtime)
+    myriadSchedulerConf.exclude group: 'org.apache.hadoop', module: '*'
 }
 
 // copies dependencies to build/libs dir
 task copyRunTimeDeps(type: Sync) {
-    from configurations.runtime
+    from configurations.myriadSchedulerConf
     into "$buildDir/libs"
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java b/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java
index 304b9d2..db4f6d9 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java
@@ -163,12 +163,13 @@ public class Main {
         NMProfileManager profileManager = injector.getInstance(NMProfileManager.class);
         long maxCpu = Long.MIN_VALUE;
         long maxMem = Long.MIN_VALUE;
-        for (String profile : nmInstances.keySet()) {
+        for (Map.Entry<String, Integer> entry : nmInstances.entrySet()) {
+          String profile = entry.getKey();
           NMProfile nmProfile = profileManager.get(profile);
           if (nmProfile == null) {
             throw new RuntimeException("Invalid profile name '" + profile + "' specified
in 'nmInstances'");
           }
-          if (nmInstances.get(profile) > 0) {
+          if (entry.getValue() > 0) {
             if (nmProfile.getCpus() > maxCpu) { // find the profile with largest number
of cpus
               maxCpu = nmProfile.getCpus();
               maxMem = nmProfile.getMemory(); // use the memory from the same profile
@@ -184,8 +185,8 @@ public class Main {
     private void startNMInstances(Injector injector) {
       Map<String, Integer> nmInstances = injector.getInstance(MyriadConfiguration.class).getNmInstances();
       MyriadOperations myriadOperations = injector.getInstance(MyriadOperations.class);
-      for (String profile : nmInstances.keySet()) {
-        myriadOperations.flexUpCluster(nmInstances.get(profile), profile);
+      for (Map.Entry<String, Integer> entry : nmInstances.entrySet()) {
+        myriadOperations.flexUpCluster(entry.getValue(), entry.getKey());
       }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/myriad-scheduler/src/main/java/com/ebay/myriad/policy/LeastAMNodesFirstPolicy.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/policy/LeastAMNodesFirstPolicy.java
b/myriad-scheduler/src/main/java/com/ebay/myriad/policy/LeastAMNodesFirstPolicy.java
index 13a4f82..38b14a7 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/policy/LeastAMNodesFirstPolicy.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/policy/LeastAMNodesFirstPolicy.java
@@ -122,7 +122,7 @@ public class LeastAMNodesFirstPolicy extends BaseInterceptor implements
NodeScal
 
     private void onNodeRemoved(NodeRemovedSchedulerEvent event) {
         SchedulerNode schedulerNode = schedulerNodes.get(event.getRemovedRMNode().getNodeID().getHost());
-        if (schedulerNode.getNodeID().equals(event.getRemovedRMNode().getNodeID())) {
+        if (schedulerNode != null && schedulerNode.getNodeID().equals(event.getRemovedRMNode().getNodeID()))
{
             schedulerNodes.remove(schedulerNode.getNodeID().getHost());
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
index 07bd2c0..554f966 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/event/handlers/ResourceOffersEventHandler.java
@@ -22,6 +22,7 @@ import com.ebay.myriad.state.NodeTask;
 import com.ebay.myriad.state.SchedulerState;
 import com.lmax.disruptor.EventHandler;
 
+import java.util.Iterator;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.mesos.Protos;
 import org.apache.mesos.Protos.Offer;
@@ -75,7 +76,8 @@ public class ResourceOffersEventHandler implements EventHandler<ResourceOffersEv
         try {
             Set<Protos.TaskID> pendingTasks = schedulerState.getPendingTaskIds();
             if (CollectionUtils.isNotEmpty(pendingTasks)) {
-                for (Offer offer : offers) {
+                for (Iterator<Offer> iterator = offers.iterator(); iterator.hasNext();)
{
+                    Offer offer = iterator.next();
                     boolean offerMatch = false;
                     Protos.TaskID launchedTaskId = null;
                     for (Protos.TaskID pendingTaskId : pendingTasks) {
@@ -100,6 +102,7 @@ public class ResourceOffersEventHandler implements EventHandler<ResourceOffersEv
                             taskToLaunch.setHostname(offer.getHostname());
                             taskToLaunch.setSlaveId(offer.getSlaveId());
                             offerMatch = true;
+                            iterator.remove(); // remove the used offer from offers list
                             break;
                         }
                     }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/fgs/OfferLifecycleManager.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/fgs/OfferLifecycleManager.java
b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/fgs/OfferLifecycleManager.java
index 035bc70..d99f204 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/fgs/OfferLifecycleManager.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/fgs/OfferLifecycleManager.java
@@ -2,7 +2,6 @@ package com.ebay.myriad.scheduler.fgs;
 
 import com.ebay.myriad.scheduler.MyriadDriver;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/CompositeInterceptor.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/CompositeInterceptor.java
b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/CompositeInterceptor.java
index dfb5fcd..350aaa2 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/CompositeInterceptor.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/CompositeInterceptor.java
@@ -108,13 +108,13 @@ public class CompositeInterceptor implements YarnSchedulerInterceptor,
Intercept
   private NodeId getNodeIdForSchedulerEvent(SchedulerEvent event) {
       switch (event.getType()) {
         case NODE_ADDED:
-          return ((NodeAddedSchedulerEvent)event).getAddedRMNode().getNodeID();
+          return ((NodeAddedSchedulerEvent) event).getAddedRMNode().getNodeID();
         case NODE_REMOVED:
-          return ((NodeRemovedSchedulerEvent)event).getRemovedRMNode().getNodeID();
+          return ((NodeRemovedSchedulerEvent) event).getRemovedRMNode().getNodeID();
         case NODE_UPDATE:
-          return ((NodeUpdateSchedulerEvent)event).getRMNode().getNodeID();
+          return ((NodeUpdateSchedulerEvent) event).getRMNode().getNodeID();
         case NODE_RESOURCE_UPDATE:
-          return ((NodeResourceUpdateSchedulerEvent)event).getRMNode().getNodeID();
+          return ((NodeResourceUpdateSchedulerEvent) event).getRMNode().getNodeID();
       }
       return null;
     }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/YarnSchedulerInterceptor.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/YarnSchedulerInterceptor.java
b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/YarnSchedulerInterceptor.java
index b3e91c1..e7f1260 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/YarnSchedulerInterceptor.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/scheduler/yarn/interceptor/YarnSchedulerInterceptor.java
@@ -16,6 +16,9 @@ import java.io.IOException;
  */
 public interface YarnSchedulerInterceptor {
 
+    /**
+     * Filters the method callbacks.
+     */
     interface CallBackFilter {
       /**
        * Method to determine if any other methods in {@link YarnSchedulerInterceptor}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/ef328d15/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java
b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java
index c4f9969..71b6f3b 100644
--- a/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java
+++ b/myriad-scheduler/src/test/java/com/ebay/myriad/scheduler/TestMyriadScheduler.java
@@ -1,11 +1,7 @@
 package com.ebay.myriad.scheduler;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-import com.ebay.myriad.scheduler.fgs.NodeStore;
+import com.ebay.myriad.scheduler.yarn.MyriadFairScheduler;
 import java.io.IOException;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -22,9 +18,7 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.ebay.myriad.Main;
-import com.ebay.myriad.scheduler.yarn.MyriadFairScheduler;
-import com.google.inject.Injector;
+import static org.junit.Assert.assertEquals;
 
 /**
  * Tests myriad scheduler.
@@ -79,11 +73,7 @@ public class TestMyriadScheduler {
   }
 
   @Test
-  public void testNodeStore() throws Exception {
-    Injector injector = Main.getInjector();
-    // TODO (Kannan Rajah) Find a better way to inject instances
-    NodeStore nodeStore = injector.getInstance(NodeStore.class);
-
+  public void testClusterMemory() throws Exception {
     // Add a node
     RMNode node1 =
         MockNodes
@@ -92,8 +82,6 @@ public class TestMyriadScheduler {
     scheduler.handle(nodeEvent1);
     assertEquals(1024, scheduler.getClusterResource().getMemory());
 
-    assertNotNull(nodeStore.getNode("127.0.0.1"));
-
     // Add another node
     RMNode node2 =
         MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2");


Mime
View raw message