fluo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [fluo-yarn] branch master updated: fixes apache/fluo#671 made reserved mem configurable (#15)
Date Mon, 08 Jan 2018 20:17:03 GMT
This is an automated email from the ASF dual-hosted git repository.

mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-yarn.git


The following commit(s) were added to refs/heads/master by this push:
     new 2d1643a  fixes apache/fluo#671 made reserved mem configurable (#15)
2d1643a is described below

commit 2d1643aef0e1390dbc26da1388cdbc7d69d20c6a
Author: Keith Turner <keith@deenlo.com>
AuthorDate: Mon Jan 8 15:17:01 2018 -0500

    fixes apache/fluo#671 made reserved mem configurable (#15)
---
 .../java/org/apache/fluo/yarn/core/FluoYarnEnv.java  | 20 ++++++++++++++++++++
 .../org/apache/fluo/yarn/core/FluoYarnLauncher.java  | 12 ++++++++++++
 .../apache/fluo/yarn/core/FluoYarnProperties.java    |  4 ++++
 distribution/conf/fluo-yarn.properties               | 15 ++++++++++-----
 distribution/lib/fetch.sh                            | 20 ++++++++++----------
 pom.xml                                              |  2 +-
 6 files changed, 57 insertions(+), 16 deletions(-)

diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
index c5a8629..15e77b0 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnEnv.java
@@ -116,6 +116,16 @@ public class FluoYarnEnv {
         FluoYarnProperties.WORKER_MAX_MEMORY_MB_DEFAULT));
   }
 
+  public String getWorkerReservedMemory() {
+    String rmem = props.getProperty(FluoYarnProperties.WORKER_RESERVED_MEMORY_MB_PROP, null);
+    if (rmem == null) {
+      return rmem;
+    }
+
+    Preconditions.checkArgument(Integer.parseInt(rmem) > 0);
+    return rmem;
+  }
+
   public int getOracleCores() {
     return Integer.valueOf(props.getProperty(FluoYarnProperties.ORACLE_NUM_CORES_PROP,
         FluoYarnProperties.ORACLE_NUM_CORES_DEFAULT));
@@ -131,6 +141,16 @@ public class FluoYarnEnv {
         FluoYarnProperties.ORACLE_MAX_MEMORY_MB_DEFAULT));
   }
 
+  public String getOracleReservedMemory() {
+    String rmem = props.getProperty(FluoYarnProperties.ORACLE_RESERVED_MEMORY_MB_PROP, null);
+    if (rmem == null) {
+      return rmem;
+    }
+
+    Preconditions.checkArgument(Integer.parseInt(rmem) > 0);
+    return rmem;
+  }
+
   public String getYarnQueue() {
     return props.getProperty(FluoYarnProperties.QUEUE_PROP, null);
   }
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
index e79f9dc..056604f 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnLauncher.java
@@ -17,7 +17,9 @@ package org.apache.fluo.yarn.core;
 
 import java.io.File;
 import java.util.Collection;
+import java.util.Collections;
 
+import org.apache.twill.api.Configs;
 import org.apache.twill.api.ResourceReport;
 import org.apache.twill.api.ResourceSpecification;
 import org.apache.twill.api.TwillApplication;
@@ -129,6 +131,16 @@ public class FluoYarnLauncher {
       preparer.setSchedulerQueue(env.getYarnQueue());
     }
 
+    if (env.getWorkerReservedMemory() != null) {
+      preparer.withConfiguration(WORKER_ID, Collections
+          .singletonMap(Configs.Keys.JAVA_RESERVED_MEMORY_MB, env.getWorkerReservedMemory()));
+    }
+
+    if (env.getOracleReservedMemory() != null) {
+      preparer.withConfiguration(ORACLE_ID, Collections
+          .singletonMap(Configs.Keys.JAVA_RESERVED_MEMORY_MB, env.getOracleReservedMemory()));
+    }
+
     TwillController controller = preparer.start();
 
     ResourceReport report = controller.getResourceReport();
diff --git a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
index 990afa7..c707928 100644
--- a/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
+++ b/core/src/main/java/org/apache/fluo/yarn/core/FluoYarnProperties.java
@@ -35,6 +35,8 @@ public class FluoYarnProperties {
   // Worker properties
   public static final String WORKER_INSTANCES_PROP = YARN_PREFIX + ".worker.instances";
   public static final String WORKER_MAX_MEMORY_MB_PROP = YARN_PREFIX + ".worker.max.memory.mb";
+  public static final String WORKER_RESERVED_MEMORY_MB_PROP =
+      YARN_PREFIX + ".worker.reserved.memory.mb";
   public static final String WORKER_NUM_CORES_PROP = YARN_PREFIX + ".worker.num.cores";
   public static final String WORKER_INSTANCES_DEFAULT = "1";
   public static final String WORKER_MAX_MEMORY_MB_DEFAULT = "1024";
@@ -43,6 +45,8 @@ public class FluoYarnProperties {
   // Oracle properties
   public static final String ORACLE_INSTANCES_PROP = YARN_PREFIX + ".oracle.instances";
   public static final String ORACLE_MAX_MEMORY_MB_PROP = YARN_PREFIX + ".oracle.max.memory.mb";
+  public static final String ORACLE_RESERVED_MEMORY_MB_PROP =
+      YARN_PREFIX + ".oracle.reserved.memory.mb";
   public static final String ORACLE_NUM_CORES_PROP = YARN_PREFIX + ".oracle.num.cores";
   public static final String ORACLE_INSTANCES_DEFAULT = "1";
   public static final String ORACLE_MAX_MEMORY_MB_DEFAULT = "512";
diff --git a/distribution/conf/fluo-yarn.properties b/distribution/conf/fluo-yarn.properties
index c2f3546..f2b1705 100644
--- a/distribution/conf/fluo-yarn.properties
+++ b/distribution/conf/fluo-yarn.properties
@@ -43,6 +43,9 @@
 ## Max memory of Oracle yarn containers (in MB)
 #fluo.yarn.oracle.max.memory.mb=512
 
+## Reserved memory.  Behaves same as worker property, so see its docs.
+#fluo.yarn.oracle.reserved.memory.mb=200
+
 ## Number of Oracle virtual cores
 #fluo.yarn.oracle.num.cores=1
 
@@ -51,12 +54,14 @@
 ## Number of Worker instances
 #fluo.yarn.worker.instances=1
 
-## Max memory of worker YARN containers (in MB). If YARN is killing worker processes consider
-## increasing twill.java.reserved.memory.mb (which defaults to 200 and is set in yarn-site.xml).
-## The twill.java.reserved.memory.mb config determines the gap between the YARN memory limit
set
-## below and the java -Xmx setting.  For example, if max memory is 1024 and twill reserved
memory
-## is 200, the java -Xmx setting will be 1024-200 = 824 MB.
+## Max memory of worker YARN containers (in MB).
 #fluo.yarn.worker.max.memory.mb=1024
 
+## This property is used to compute the Java heap size by subtracting it from the max memory
+## property.  For example, if max memory is 1024 and reserved memory is 200, then java -Xmx
setting
+## will be 1024-200 = 824 MB. If YARN is killing workers because too much memory is used,
 then
+## consider increasing this property. 
+#fluo.yarn.worker.reserved.memory.mb=200
+
 ## Number of worker virtual cores
 #fluo.yarn.worker.num.cores=1
diff --git a/distribution/lib/fetch.sh b/distribution/lib/fetch.sh
index 0881187..2c351f0 100755
--- a/distribution/lib/fetch.sh
+++ b/distribution/lib/fetch.sh
@@ -17,7 +17,7 @@ lib_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 maven_prefix=https://repo1.maven.org/maven2
 
 function download {
-  IFS=':' read -ra DEP <<< "$1" 
+  IFS=':' read -ra DEP <<< "$1"
   dir=$lib_dir/
   if [ -n "$2" ]; then
     dir=$lib_dir/$2
@@ -35,7 +35,7 @@ function download {
 
   if [ -f $dir/$fn ]; then
     echo "SUCCESS: Dependency exists - $dir/$fn"
-  else 
+  else
     wget -q $download_url -P $dir
     if [ $? == 0 ]; then
       echo "SUCCESS: Dependency downloaded from $download_url"
@@ -57,14 +57,14 @@ download com.yammer.metrics:metrics-annotation:jar:2.2.0
 download com.yammer.metrics:metrics-core:jar:2.2.0
 download net.sf.jopt-simple:jopt-simple:jar:3.2
 download org.apache.kafka:kafka_2.10:jar:0.8.0
-download org.apache.twill:twill-api:jar:0.11.0
-download org.apache.twill:twill-common:jar:0.11.0
-download org.apache.twill:twill-core:jar:0.11.0
-download org.apache.twill:twill-discovery-api:jar:0.11.0
-download org.apache.twill:twill-discovery-core:jar:0.11.0
-download org.apache.twill:twill-ext:jar:0.11.0
-download org.apache.twill:twill-yarn:jar:0.11.0
-download org.apache.twill:twill-zookeeper:jar:0.11.0
+download org.apache.twill:twill-api:jar:0.12.0
+download org.apache.twill:twill-common:jar:0.12.0
+download org.apache.twill:twill-core:jar:0.12.0
+download org.apache.twill:twill-discovery-api:jar:0.12.0
+download org.apache.twill:twill-discovery-core:jar:0.12.0
+download org.apache.twill:twill-ext:jar:0.12.0
+download org.apache.twill:twill-yarn:jar:0.12.0
+download org.apache.twill:twill-zookeeper:jar:0.12.0
 download org.ow2.asm:asm-all:jar:5.0.2
 download org.scala-lang:scala-compiler:jar:2.10.1
 download org.scala-lang:scala-library:jar:2.10.1
diff --git a/pom.xml b/pom.xml
index 00cda0d..ec732cb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,7 @@
     <hadoop.version>2.6.3</hadoop.version>
     <logback.version>1.1.3</logback.version>
     <slf4j.version>1.7.12</slf4j.version>
-    <twill.version>0.11.0</twill.version>
+    <twill.version>0.12.0</twill.version>
   </properties>
   <dependencyManagement>
     <dependencies>

-- 
To stop receiving notification emails like this one, please contact
['"commits@fluo.apache.org" <commits@fluo.apache.org>'].

Mime
View raw message