ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject [3/8] incubator-ignite git commit: #IGNITE-857 WIP
Date Tue, 09 Jun 2015 16:06:44 GMT
#IGNITE-857 WIP


Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/81cde9b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/81cde9b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/81cde9b7

Branch: refs/heads/yarn
Commit: 81cde9b71b3ab88023f68fa6f3bee86f20442412
Parents: b569191
Author: nikolay tikhonov <ntikhonov@gridgain.com>
Authored: Wed Jun 3 18:37:02 2015 +0300
Committer: nikolay tikhonov <ntikhonov@gridgain.com>
Committed: Wed Jun 3 18:37:02 2015 +0300

----------------------------------------------------------------------
 .../apache/ignite/yarn/ApplicationMaster.java   | 51 +++++++++++++++-----
 .../apache/ignite/yarn/IgniteYarnClient.java    | 15 ++----
 2 files changed, 45 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/81cde9b7/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
index 9ab70d4..532830c 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
@@ -19,6 +19,9 @@ package org.apache.ignite.yarn;
 
 import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.*;
 import org.apache.hadoop.yarn.api.protocolrecords.*;
 import org.apache.hadoop.yarn.api.records.*;
@@ -33,27 +36,33 @@ import java.util.*;
  * TODO
  */
 public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
-    Configuration configuration;
+    YarnConfiguration configuration;
     NMClient nmClient;
-    int numContainersToWaitFor = 5;
+    int numContainersToWaitFor = 1;
 
     public ApplicationMaster() {
         configuration = new YarnConfiguration();
+
         nmClient = NMClient.createNMClient();
         nmClient.init(configuration);
         nmClient.start();
     }
 
+    /** {@inheritDoc} */
     public void onContainersAllocated(List<Container> containers) {
         for (Container container : containers) {
             try {
                 // Launch container by create ContainerLaunchContext
-                // bin/hadoop fs -rm /user/ntikhonov/*.jar && bin/hadoop fs -copyFromLocal
./ignite-yarn.jar /user/ntikhonov
-                ContainerLaunchContext ctx =
-                        Records.newRecord(ContainerLaunchContext.class);
+                ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
+
+                final LocalResource igniteZip = Records.newRecord(LocalResource.class);
+                setupAppMasterJar(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6.zip"),
igniteZip,
+                    configuration);
+
+                ctx.setLocalResources(Collections.singletonMap("ignite", igniteZip));
                 ctx.setCommands(
                         Lists.newArrayList(
-                                "ls " +
+                                "$LOCAL_DIRS/ignite/*/bin/ignite.sh" +
                                 " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
+
                                 " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
                         ));
@@ -65,6 +74,24 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
         }
     }
 
+    /** {@inheritDoc} */
+    private static void setupAppMasterJar(Path jarPath, LocalResource appMasterJar, YarnConfiguration
conf)
+        throws Exception {
+        FileSystem fileSystem = FileSystem.get(conf);
+        jarPath = fileSystem.makeQualified(jarPath);
+
+        FileStatus jarStat = fileSystem.getFileStatus(jarPath);
+
+        appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
+        appMasterJar.setSize(jarStat.getLen());
+        appMasterJar.setTimestamp(jarStat.getModificationTime());
+        appMasterJar.setType(LocalResourceType.ARCHIVE);
+        appMasterJar.setVisibility(LocalResourceVisibility.APPLICATION);
+
+        System.out.println("Path :" + jarPath);
+    }
+
+    /** {@inheritDoc} */
     public void onContainersCompleted(List<ContainerStatus> statuses) {
         for (ContainerStatus status : statuses) {
             System.out.println("[AM] Completed container " + status.getContainerId());
@@ -74,20 +101,21 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
         }
     }
 
+    /** {@inheritDoc} */
     public void onNodesUpdated(List<NodeReport> updated) {
     }
 
-    public void onReboot() {
-    }
-
+    /** {@inheritDoc} */
     public void onShutdownRequest() {
     }
 
+    /** {@inheritDoc} */
     public void onError(Throwable t) {
     }
 
+    /** {@inheritDoc} */
     public float getProgress() {
-        return 0;
+        return 50;
     }
 
     public boolean doneWithContainers() {
@@ -101,7 +129,6 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
     public static void main(String[] args) throws Exception {
         ApplicationMaster master = new ApplicationMaster();
         master.runMainLoop();
-
     }
 
     public void runMainLoop() throws Exception {
@@ -136,6 +163,8 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
             Thread.sleep(100);
         }
 
+
+
         System.out.println("[AM] unregisterApplicationMaster 0");
         // Un-register with ResourceManager
         rmClient.unregisterApplicationMaster(

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/81cde9b7/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
index e020ef4..092aaa9 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
@@ -74,20 +74,15 @@ public class IgniteYarnClient {
         final LocalResource igniteZip = Records.newRecord(LocalResource.class);
         setupAppMasterJar(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6.zip"),
igniteZip, conf);
 
-        FileSystem fileSystem = FileSystem.get(conf);
-
-        Path path = fileSystem.makeQualified(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6/bin/ignite.sh"));
-
-        System.out.println("Path: " + path);
-        System.out.println("Path URI: " + path.toUri().toString());
-
-        amContainer.setLocalResources(new HashMap<String, LocalResource>(){{
+        amContainer.setLocalResources(new HashMap<String, LocalResource>() {{
             put("ignite-yarn.jar", appMasterJar);
-            put("ignite", igniteZip);
+            put("gridgain-community-fabric-1.0.6.zip", igniteZip);
         }});
 
+
+
         // Setup CLASSPATH for ApplicationMaster
-        Map<String, String> appMasterEnv = new HashMap<String, String>();
+        Map<String, String> appMasterEnv = new HashMap<>();
         setupAppMasterEnv(appMasterEnv, conf);
         amContainer.setEnvironment(appMasterEnv);
 


Mime
View raw message