myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mo...@apache.org
Subject [18/50] [abbrv] incubator-myriad git commit: Added ability to launch NMs at RM's startup
Date Fri, 25 Sep 2015 22:45:58 GMT
Added ability to launch NMs at RM's startup

- Added a section "nmInstances" in .yml file
- "nmInstances" allows to specify the number of NM instances of a specific profile
- Default is "medium: 1" (can be changed)
- Multiple profiles can be specified. for e.g.
  medium: 1
  small: 3
  zero: 2
- Added validation for nmInstances. RM doesn't start if
  - profile name is incorrect
  - number of NMs to launch is zero
  - largest profile has zero cpus or zero memory


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/3e47ed18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/3e47ed18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/3e47ed18

Branch: refs/heads/master
Commit: 3e47ed18fc371db04a3c54760e5d2b1eae055732
Parents: 8e355b5
Author: Santosh Marella <smarella@maprtech.com>
Authored: Fri Jul 10 17:29:12 2015 -0700
Committer: Santosh Marella <smarella@maprtech.com>
Committed: Fri Jul 10 17:29:12 2015 -0700

----------------------------------------------------------------------
 .../src/main/java/com/ebay/myriad/Main.java     | 35 ++++++++++++++++++++
 .../configuration/MyriadConfiguration.java      |  8 +++++
 .../main/resources/myriad-config-default.yml    |  4 ++-
 3 files changed, 46 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/3e47ed18/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java b/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java
index a756bb0..304b9d2 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/Main.java
@@ -23,6 +23,7 @@ import com.ebay.myriad.health.MesosDriverHealthCheck;
 import com.ebay.myriad.health.MesosMasterHealthCheck;
 import com.ebay.myriad.health.ZookeeperHealthCheck;
 import com.ebay.myriad.scheduler.MyriadDriverManager;
+import com.ebay.myriad.scheduler.MyriadOperations;
 import com.ebay.myriad.scheduler.NMProfile;
 import com.ebay.myriad.scheduler.NMProfileManager;
 import com.ebay.myriad.scheduler.Rebalancer;
@@ -94,11 +95,13 @@ public class Main {
         initWebApp(injector);
         initHealthChecks(injector);
         initProfiles(injector);
+        validateNMInstances(injector);
         initDisruptors(injector);
 
         initRebalancerService(cfg, injector);
         initTerminatorService(injector);
         startMesosDriver(injector);
+        startNMInstances(injector);
     }
 
     private void startMesosDriver(Injector injector) {
@@ -154,6 +157,38 @@ public class Main {
         }
     }
 
+    private void validateNMInstances(Injector injector) {
+        LOGGER.info("Validating nmInstances..");
+        Map<String, Integer> nmInstances = injector.getInstance(MyriadConfiguration.class).getNmInstances();
+        NMProfileManager profileManager = injector.getInstance(NMProfileManager.class);
+        long maxCpu = Long.MIN_VALUE;
+        long maxMem = Long.MIN_VALUE;
+        for (String profile : nmInstances.keySet()) {
+          NMProfile nmProfile = profileManager.get(profile);
+          if (nmProfile == null) {
+            throw new RuntimeException("Invalid profile name '" + profile + "' specified
in 'nmInstances'");
+          }
+          if (nmInstances.get(profile) > 0) {
+            if (nmProfile.getCpus() > maxCpu) { // find the profile with largest number
of cpus
+              maxCpu = nmProfile.getCpus();
+              maxMem = nmProfile.getMemory(); // use the memory from the same profile
+            }
+          }
+        }
+        if (maxCpu <= 0 || maxMem <= 0) {
+          throw new RuntimeException("Please configure 'nmInstances' with at least one instance/profile
" 
+              + "with non-zero cpu/mem resources.");
+        }
+    }
+
+    private void startNMInstances(Injector injector) {
+      Map<String, Integer> nmInstances = injector.getInstance(MyriadConfiguration.class).getNmInstances();
+      MyriadOperations myriadOperations = injector.getInstance(MyriadOperations.class);
+      for (String profile : nmInstances.keySet()) {
+        myriadOperations.flexUpCluster(nmInstances.get(profile), profile);
+      }
+    }
+
     private void initTerminatorService(Injector injector) {
         LOGGER.info("Initializing Terminator");
         terminatorService = Executors.newScheduledThreadPool(1);

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/3e47ed18/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java
b/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java
index 0220630..4027189 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java
@@ -104,6 +104,10 @@ public class MyriadConfiguration {
   private Map<String, Map<String, String>> profiles;
 
   @JsonProperty
+  @NotEmpty
+  private Map<String, Integer> nmInstances;
+
+  @JsonProperty
   private Boolean rebalancer;
 
   @JsonProperty
@@ -175,6 +179,10 @@ public class MyriadConfiguration {
     return profiles;
   }
 
+  public Map<String, Integer> getNmInstances() {
+    return nmInstances;
+  }
+
   public Boolean isRebalancer() {
     return rebalancer != null ? rebalancer : DEFAULT_REBALANCER;
   }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/3e47ed18/myriad-scheduler/src/main/resources/myriad-config-default.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/resources/myriad-config-default.yml b/myriad-scheduler/src/main/resources/myriad-config-default.yml
index 02b6c7d..be74043 100644
--- a/myriad-scheduler/src/main/resources/myriad-config-default.yml
+++ b/myriad-scheduler/src/main/resources/myriad-config-default.yml
@@ -12,7 +12,7 @@ zkServers: localhost:2181
 zkTimeout: 20000
 restApiPort: 8192
 profiles:
-  zero:
+  zero:  # NMs launched with this profile dynamically obtain cpu/mem from Mesos
     cpu: 0
     mem: 0
   small:
@@ -24,6 +24,8 @@ profiles:
   large:
     cpu: 4
     mem: 4096
+nmInstances: # NMs to start with. Requires at least 1 NM with a non-zero profile.
+  medium: 1 # <profile_name : instances>
 rebalancer: false
 nodemanager:
   jvmMaxMemoryMB: 1024


Mime
View raw message