myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dar...@apache.org
Subject incubator-myriad git commit: Added the ability to specify a URI to pull configuration data from, also added to REST endpoints: /api/config.tgz and /api/binary.tgz which provide an easy download for the hadoop configs and the hadoop binarys. Configuratio
Date Thu, 12 May 2016 02:09:48 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master bcd1fc8c6 -> 5118cff3e


Added the ability to specify a URI to pull configuration data from, also added to REST endpoints:
/api/config.tgz and /api/binary.tgz which provide an easy download for the hadoop configs
and the hadoop binarys.  Configuration is done by assigning:

servedConfigPath: dist/config.tgz
servedBinaryPath: dist/hadoop-2.7.0.tgz

and

executor:
  configUri: http://127.0.0.1:8192/api/artifacts/config.tgz
  nodemanagerUri: http://127.0.0.1:8192/api/artifacts/binary.tgz

Able to use,
yarnEnvironment:
  HADOOP_CONF_DIR=$MESOS_SANDBOX/config
  YARN_HOME=$MESOS_SANDBOX/hadoop-2.7.0

JIRA:
  [Myriad-189] https://issues.apache.org/jira/browse/MYRIAD-189
Pull Request:
  Closes #68
Author:
  DarinJ <darinj@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/5118cff3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/5118cff3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/5118cff3

Branch: refs/heads/master
Commit: 5118cff3e0b3ba2e66023cd9a2840e74c546a6db
Parents: bcd1fc8
Author: darinj <darinj.work@gmail.com>
Authored: Mon May 2 23:09:49 2016 -0400
Committer: darinj <darinj@apache.org>
Committed: Wed May 11 21:56:45 2016 -0400

----------------------------------------------------------------------
 .../apache/myriad/api/ArtifactsResource.java    | 78 ++++++++++++++++++++
 .../configuration/MyriadConfiguration.java      | 13 ++++
 .../MyriadExecutorConfiguration.java            |  8 ++
 .../apache/myriad/scheduler/TaskFactory.java    | 11 ++-
 .../myriad/webapp/MyriadServletModule.java      |  3 +-
 .../main/resources/myriad-config-default.yml    |  4 +
 6 files changed, 113 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/api/ArtifactsResource.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/api/ArtifactsResource.java b/myriad-scheduler/src/main/java/org/apache/myriad/api/ArtifactsResource.java
new file mode 100644
index 0000000..769abe0
--- /dev/null
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/api/ArtifactsResource.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.api;
+
+import com.codahale.metrics.annotation.Timed;
+import org.apache.myriad.configuration.MyriadConfiguration;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.File;
+
+/**
+ * Defines the REST API to the Myriad configuration.
+ */
+@Path("/artifacts")
+@Produces(MediaType.APPLICATION_OCTET_STREAM)
+public class ArtifactsResource {
+  private String myriadEtc;
+  private String myriadBinary;
+
+  @Inject
+  public ArtifactsResource(MyriadConfiguration cfg) {
+    myriadEtc = cfg.getServedConfigPath().or("");
+    myriadBinary = cfg.getServedBinaryPath().or("");
+  }
+
+  @Timed
+  @GET
+  @Path("/config.tgz")
+  public Response getConfig() throws InterruptedException {
+    File file = new File(myriadEtc);
+    Response.ResponseBuilder response;
+    if (file.exists()) {
+      response = Response.ok((Object) file);
+      response.header("Content-Disposition", "attachment; filename=" + file.getName());
+    } else {
+      response = Response.status(Response.Status.BAD_REQUEST)
+          .entity("Path does not exist");
+    }
+    return response.build();
+  }
+
+  @Timed
+  @GET
+  @Path("/binary.tgz")
+  public Response getBinary() throws InterruptedException {
+    File file = new File(myriadBinary);
+    Response.ResponseBuilder response;
+    if (file.exists()) {
+      response = Response.ok((Object) file);
+      response.header("Content-Disposition", "attachment; filename=" + file.getName());
+    } else {
+      response = Response.status(Response.Status.BAD_REQUEST)
+          .entity("Path does not exist");
+    }
+    return response.build();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
index 716a20a..f65bb9a 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
@@ -179,6 +179,11 @@ public class MyriadConfiguration {
   @JsonProperty
   private String mesosAuthenticationSecretFilename;
 
+  @JsonProperty
+  private String servedConfigPath;
+
+  @JsonProperty
+  private String servedBinaryPath;
 
   public MyriadConfiguration() {
   }
@@ -278,4 +283,12 @@ public class MyriadConfiguration {
     return mesosAuthenticationPrincipal;
   }
 
+  public Optional<String> getServedConfigPath() {
+    return Optional.fromNullable(servedConfigPath);
+  }
+
+  public Optional<String> getServedBinaryPath() {
+    return Optional.fromNullable(servedBinaryPath);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
index 6afc086..ea98ef7 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
@@ -44,6 +44,10 @@ public class MyriadExecutorConfiguration {
   @JsonSerialize(using = OptionalSerializerString.class)
   private String nodeManagerUri;
 
+  @JsonProperty
+  @JsonSerialize(using = OptionalSerializerString.class)
+  private String configUri;
+
   /**
    * Download URL for JRE.
    * Ex: jvmUri: http://www.apache.org/myriad/jre-1.8.99.tar.gz
@@ -66,6 +70,10 @@ public class MyriadExecutorConfiguration {
     return Optional.fromNullable(nodeManagerUri);
   }
 
+  public Optional<String> getConfigUri() {
+    return Optional.fromNullable(configUri);
+  }
+
   public Optional<String> getJvmUri() {
     return Optional.fromNullable(jvmUri);
   }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
index 9ed54a7..92e4cf7 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
@@ -133,6 +133,12 @@ public interface TaskFactory {
         commandInfo.addUris(jvmUri);
       }
 
+      if (myriadExecutorConfiguration.getConfigUri().isPresent()) {
+        String configURI = myriadExecutorConfiguration.getConfigUri().get();
+        LOGGER.info("Getting Hadoop distribution from: {}", configURI);
+        commandInfo.addUris(URI.newBuilder().setValue(configURI).build());
+      }
+
       if (myriadExecutorConfiguration.getNodeManagerUri().isPresent()) {
         //Both FrameworkUser and FrameworkSuperuser to get all of the directory permissions
correct.
         if (!(cfg.getFrameworkUser().isPresent() && cfg.getFrameworkSuperUser().isPresent()))
{
@@ -143,14 +149,14 @@ public interface TaskFactory {
 
         //get the nodemanagerURI
         //We're going to extract ourselves, so setExtract is false
-        LOGGER.info("Getting Hadoop distribution from:" + nodeManagerUri);
+        LOGGER.info("Getting Hadoop distribution from: {}", nodeManagerUri);
         URI nmUri = URI.newBuilder().setValue(nodeManagerUri).setExtract(false).build();
 
         //get configs directly from resource manager
         String configUrlString = clGenerator.getConfigurationUrl();
         LOGGER.info("Getting config from:" + configUrlString);
         URI configUri = URI.newBuilder().setValue(configUrlString).build();
-        LOGGER.info("Slave will execute command:" + cmd);
+        LOGGER.info("Slave will execute command: {}",  cmd);
         commandInfo.addUris(nmUri).addUris(configUri).setValue("echo \"" + cmd + "\";" +
cmd);
         commandInfo.setUser(cfg.getFrameworkSuperUser().get());
 
@@ -162,7 +168,6 @@ public interface TaskFactory {
           commandInfo.setUser(cfg.getFrameworkUser().get());
         }
       }
-
       return commandInfo.build();
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java
b/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java
index 3aa3737..82d2bae 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java
@@ -25,6 +25,7 @@ import org.apache.myriad.api.ClustersResource;
 import org.apache.myriad.api.ConfigurationResource;
 import org.apache.myriad.api.ControllerResource;
 import org.apache.myriad.api.SchedulerStateResource;
+import org.apache.myriad.api.ArtifactsResource;
 import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider;
 
 /**
@@ -38,7 +39,7 @@ public class MyriadServletModule extends ServletModule {
     bind(ConfigurationResource.class);
     bind(SchedulerStateResource.class);
     bind(ControllerResource.class);
-
+    bind(ArtifactsResource.class);
     bind(GuiceContainer.class);
     bind(JacksonJaxbJsonProvider.class).in(Scopes.SINGLETON);
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/resources/myriad-config-default.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/resources/myriad-config-default.yml b/myriad-scheduler/src/main/resources/myriad-config-default.yml
index a90de84..a2c143d 100644
--- a/myriad-scheduler/src/main/resources/myriad-config-default.yml
+++ b/myriad-scheduler/src/main/resources/myriad-config-default.yml
@@ -29,6 +29,8 @@ nativeLibrary: /usr/local/lib/libmesos.so
 zkServers: localhost:2181
 zkTimeout: 20000
 restApiPort: 8192
+#servedConfigPath: dist/config.tgz
+#servedBinaryPath: dist/hadoop-2.6.0.tgz
 profiles:
   zero:  # NMs launched with this profile dynamically obtain cpu/mem from Mesos
     cpu: 0
@@ -55,10 +57,12 @@ executor:
   path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar
   #The following should be used for a remotely distributed URI, hdfs assumed but other URI
types valid.
   #nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz
+  #configUri: http://127.0.0.1/api/arifacts/config.tgz
   #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar
   #jvmUri: https://downloads.mycompany.com/java/jre-7u76-linux-x64.tar.gz
 yarnEnvironment:
   YARN_HOME: /usr/local/hadoop
+  #HADOOP_CONF_DIR=config
   #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set
   #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary
   #JAVA_HOME: jre1.7.0_76 # Path to JRE distribution, relative to sandbox directory


Mime
View raw message