ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ntikho...@apache.org
Subject ignite git commit: Fixed IGNITE-2419 Ignite on YARN do not handle memory overhead. This closes #414.
Date Mon, 01 Feb 2016 04:35:26 GMT
Repository: ignite
Updated Branches:
  refs/heads/master 59691291b -> 1945b9884


Fixed IGNITE-2419 Ignite on YARN do not handle memory overhead. This closes #414.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1945b988
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1945b988
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1945b988

Branch: refs/heads/master
Commit: 1945b98849cb52fc297c726fa79a270706135581
Parents: 5969129
Author: Edouard Chevalier <edouard.chevalier@techmydata.net>
Authored: Mon Feb 1 07:34:59 2016 +0300
Committer: Tikhonov Nikolay <tikhonovnicolay@gmail.com>
Committed: Mon Feb 1 07:34:59 2016 +0300

----------------------------------------------------------------------
 .../apache/ignite/yarn/ApplicationMaster.java   |  12 +-
 .../apache/ignite/yarn/ClusterProperties.java   | 144 +++++++++++--------
 .../yarn/IgniteApplicationMasterSelfTest.java   |  52 +++++++
 3 files changed, 140 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
index 755e4e4..b9ab02d 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
@@ -137,8 +137,8 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
                             + "cp -r ./libs/* ./ignite/*/libs/ || true && "
                             + "./ignite/*/bin/ignite.sh "
                             + "./ignite-config.xml"
-                            + " -J-Xmx" + c.getResource().getMemory() + "m"
-                            + " -J-Xms" + c.getResource().getMemory() + "m"
+                            + " -J-Xmx" + ((int)props.memoryPerNode()) + "m"
+                            + " -J-Xms" + ((int)props.memoryPerNode()) + "m"
                             + IgniteYarnUtils.YARN_LOG_OUT
                         ));
 
@@ -178,7 +178,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
 
         // Check that slave satisfies min requirements.
         if (cont.getResource().getVirtualCores() < props.cpusPerNode()
-            || cont.getResource().getMemory() < props.memoryPerNode()) {
+            || cont.getResource().getMemory() < props.totalMemoryPerNode()) {
             log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0},
cpu: {1}, mem: {2}",
                 new Object[]{cont.getNodeId().getHost(), cont.getResource().getVirtualCores(),
                    cont.getResource().getMemory()});
@@ -291,7 +291,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
                     // Resource requirements for worker containers.
                     Resource capability = Records.newRecord(Resource.class);
 
-                    capability.setMemory((int)props.memoryPerNode());
+                    capability.setMemory((int)props.totalMemoryPerNode());
                     capability.setVirtualCores((int)props.cpusPerNode());
 
                     for (int i = 0; i < props.instances() - runningCnt; ++i) {
@@ -302,7 +302,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
                         rmClient.addContainerRequest(containerAsk);
 
                         log.log(Level.INFO, "Making request. Memory: {0}, cpu {1}.",
-                            new Object[]{props.memoryPerNode(), props.cpusPerNode()});
+                            new Object[]{props.totalMemoryPerNode(), props.cpusPerNode()});
                     }
                 }
 
@@ -329,7 +329,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
     private boolean checkAvailableResource() {
         Resource availableRes = rmClient.getAvailableResources();
 
-        return availableRes == null || availableRes.getMemory() >= props.memoryPerNode()
+        return availableRes == null || availableRes.getMemory() >= props.totalMemoryPerNode()
             && availableRes.getVirtualCores() >= props.cpusPerNode();
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
index d040e9f..647aef2 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java
@@ -48,6 +48,12 @@ public class ClusterProperties {
 
     /** */
     public static final double DEFAULT_MEM_PER_NODE = 2048;
+    
+    /**
+     * The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE,
+     * with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE.
+     */
+    public static final double DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE = 384;
 
     /** Cluster name. */
     private String clusterName = DEFAULT_CLUSTER_NAME;
@@ -63,6 +69,12 @@ public class ClusterProperties {
 
     /** Memory limit. */
     private double memPerNode = DEFAULT_MEM_PER_NODE;
+    
+    /** */
+    public static final String IGNITE_MEMORY_OVERHEAD_PER_NODE = "IGNITE_MEMORY_OVERHEAD_PER_NODE";
+
+    /** Memory over head to request yarn. */
+    private double memOverHeadPerNode = 0;
 
     /** */
     public static final String IGNITE_NODE_COUNT = "IGNITE_NODE_COUNT";
@@ -185,7 +197,30 @@ public class ClusterProperties {
     }
 
     /**
-     * @return instance count limit.
+     * @return Memory overhead for requested memory.
+     */
+    public double memoryOverHeadPerNode() {
+		return memOverHeadPerNode;
+	}
+
+    /**
+     * Sets memory overhead requested to YARN.
+     *
+     * @param memOverHeadPerNode Memory over head per node.
+     */
+	public void memoryOverHeadPerNode(double memOverHeadPerNode) {
+		this.memOverHeadPerNode = memOverHeadPerNode;
+	}
+	
+	/**
+	 * @return Provide the total memory requested to ResourceManagers (memoryPerNode + memoryOverheadPerNode).
+	 */
+	public double totalMemoryPerNode(){
+		return memoryPerNode() + memoryOverHeadPerNode();
+	}
+
+	/**
+     * @return Instance count limit.
      */
     public double instances() {
         return nodeCnt;
@@ -278,7 +313,50 @@ public class ClusterProperties {
     }
 
     /**
-     * @param config path to config file.
+     * Instantiate a ClusterProperties from a set of properties.
+     *
+     * @param props If {@code null} will be used system properties.
+     * @return Cluster properties.
+     */
+    private static ClusterProperties fromProperties(Properties props) {
+    	ClusterProperties prop = new ClusterProperties();
+
+        prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, props, DEFAULT_CLUSTER_NAME);
+
+        prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, props, DEFAULT_CPU_PER_NODE);
+        prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, props, DEFAULT_MEM_PER_NODE);
+        // The minimum memory overhead: overhead is by default 0.1* MEMORY_PER_NODE,
+        // with a minimum of DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE
+        prop.memOverHeadPerNode = getDoubleProperty(IGNITE_MEMORY_OVERHEAD_PER_NODE, props,
+            Math.max( 0.1 * prop.memPerNode, DEFAULT_MINIMUM_MEM_OVERHEAD_PER_NODE));
+        prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT);
+
+        prop.igniteUrl = getStringProperty(IGNITE_URL, props, null);
+        prop.ignitePath = getStringProperty(IGNITE_PATH, props, null);
+        prop.licencePath = getStringProperty(LICENCE_PATH, props, null);
+        prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null);
+        prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, props, DEFAULT_IGNITE_WORK_DIR);
+        prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, props, DEFAULT_IGNITE_LOCAL_WORK_DIR);
+        prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, props, DEFAULT_IGNITE_RELEASES_DIR);
+        prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null);
+        prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null);
+
+        String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, props, null);
+
+        if (pattern != null) {
+            try {
+                prop.hostnameConstraint = Pattern.compile(pattern);
+            }
+            catch (PatternSyntaxException e) {
+                log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It
will be ignore.", e);
+            }
+        }
+
+        return prop;
+    }
+    
+    /**
+     * @param config Path to config file.
      * @return Cluster configuration.
      */
     public static ClusterProperties from(String config) {
@@ -291,36 +369,7 @@ public class ClusterProperties {
                 props.load(new FileInputStream(config));
             }
 
-            ClusterProperties prop = new ClusterProperties();
-
-            prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, props, DEFAULT_CLUSTER_NAME);
-
-            prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, props, DEFAULT_CPU_PER_NODE);
-            prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, props, DEFAULT_MEM_PER_NODE);
-            prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT);
-
-            prop.igniteUrl = getStringProperty(IGNITE_URL, props, null);
-            prop.ignitePath = getStringProperty(IGNITE_PATH, props, null);
-            prop.licencePath = getStringProperty(LICENCE_PATH, props, null);
-            prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null);
-            prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, props, DEFAULT_IGNITE_WORK_DIR);
-            prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, props, DEFAULT_IGNITE_LOCAL_WORK_DIR);
-            prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, props, DEFAULT_IGNITE_RELEASES_DIR);
-            prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, props, null);
-            prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, props, null);
-
-            String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, props, null);
-
-            if (pattern != null) {
-                try {
-                    prop.hostnameConstraint = Pattern.compile(pattern);
-                }
-                catch (PatternSyntaxException e) {
-                    log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern.
It will be ignore.", e);
-                }
-            }
-
-            return prop;
+            return fromProperties(props);
         }
         catch (IOException e) {
             throw new RuntimeException(e);
@@ -331,36 +380,7 @@ public class ClusterProperties {
      * @return Cluster configuration.
      */
     public static ClusterProperties from() {
-        ClusterProperties prop = new ClusterProperties();
-
-        prop.clusterName = getStringProperty(IGNITE_CLUSTER_NAME, null, DEFAULT_CLUSTER_NAME);
-
-        prop.cpuPerNode = getDoubleProperty(IGNITE_RUN_CPU_PER_NODE, null, DEFAULT_CPU_PER_NODE);
-        prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, null, DEFAULT_MEM_PER_NODE);
-        prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, null, DEFAULT_IGNITE_NODE_COUNT);
-
-        prop.igniteUrl = getStringProperty(IGNITE_URL, null, null);
-        prop.ignitePath = getStringProperty(IGNITE_PATH, null, null);
-        prop.licencePath = getStringProperty(LICENCE_PATH, null, null);
-        prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, null, null);
-        prop.igniteWorkDir = getStringProperty(IGNITE_WORKING_DIR, null, DEFAULT_IGNITE_WORK_DIR);
-        prop.igniteLocalWorkDir = getStringProperty(IGNITE_LOCAL_WORK_DIR, null, DEFAULT_IGNITE_LOCAL_WORK_DIR);
-        prop.igniteReleasesDir = getStringProperty(IGNITE_RELEASES_DIR, null, DEFAULT_IGNITE_RELEASES_DIR);
-        prop.igniteCfg = getStringProperty(IGNITE_CONFIG_XML, null, null);
-        prop.userLibs = getStringProperty(IGNITE_USERS_LIBS, null, null);
-
-        String pattern = getStringProperty(IGNITE_HOSTNAME_CONSTRAINT, null, null);
-
-        if (pattern != null) {
-            try {
-                prop.hostnameConstraint = Pattern.compile(pattern);
-            }
-            catch (PatternSyntaxException e) {
-                log.log(Level.WARNING, "IGNITE_HOSTNAME_CONSTRAINT has invalid pattern. It
will be ignore.", e);
-            }
-        }
-
-        return prop;
+        return fromProperties(null);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/1945b988/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
index 97f6a12..1190313 100644
--- a/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
+++ b/modules/yarn/src/test/java/org/apache/ignite/yarn/IgniteApplicationMasterSelfTest.java
@@ -103,6 +103,58 @@ public class IgniteApplicationMasterSelfTest extends TestCase {
             assertEquals(1024, req.getCapability().getMemory());
         }
     }
+    
+    /**
+     * Tests whether memory overhead is allocated within container memory.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMemoryOverHeadAllocation() throws Exception {
+        appMaster.setRmClient(rmMock);
+        appMaster.setNmClient(new NMMock());
+
+        props.cpusPerNode(2);
+        props.memoryPerNode(1024);
+        props.memoryOverHeadPerNode(512);
+        props.instances(3);
+
+        Thread thread = runAppMaster(appMaster);
+
+        List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1,
1000);
+
+        interruptedThread(thread);
+
+        assertEquals(3, contRequests.size());
+
+        for (AMRMClient.ContainerRequest req : contRequests) {
+            assertEquals(2, req.getCapability().getVirtualCores());
+            assertEquals(1024 + 512, req.getCapability().getMemory());
+        }
+    }
+
+    /**
+     * Tests whether memory overhead prevents from allocating container.
+     *
+     * @throws Exception If failed.
+     */
+     public void testMemoryOverHeadPreventAllocation() throws Exception {
+        rmMock.availableRes(new MockResource(1024, 2));
+        appMaster.setRmClient(rmMock);
+        appMaster.setNmClient(new NMMock());
+
+        props.cpusPerNode(2);
+        props.memoryPerNode(1024);
+        props.memoryOverHeadPerNode(512);
+        props.instances(3);
+
+        Thread thread = runAppMaster(appMaster);
+
+        List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1,
1000);
+
+        interruptedThread(thread);
+
+        assertEquals(0, contRequests.size());
+     }
 
     /**
      * @throws Exception If failed.


Mime
View raw message