incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tra...@apache.org
Subject svn commit: r1434845 - in /incubator/hcatalog/branches/branch-0.5: ./ src/docs/src/documentation/content/xdocs/ webhcat/svr/src/main/config/ webhcat/svr/src/main/java/org/apache/hcatalog/templeton/
Date Thu, 17 Jan 2013 18:22:35 GMT
Author: travis
Date: Thu Jan 17 18:22:35 2013
New Revision: 1434845

URL: http://svn.apache.org/viewvc?rev=1434845&view=rev
Log:
HCATALOG-587 webhcat controller map task takes too much memory

Modified:
    incubator/hcatalog/branches/branch-0.5/CHANGES.txt
    incubator/hcatalog/branches/branch-0.5/src/docs/src/documentation/content/xdocs/configuration.xml
    incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/config/webhcat-default.xml
    incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
    incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java

Modified: incubator/hcatalog/branches/branch-0.5/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.5/CHANGES.txt?rev=1434845&r1=1434844&r2=1434845&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.5/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.5/CHANGES.txt Thu Jan 17 18:22:35 2013
@@ -162,6 +162,8 @@ Release 0.5.0 - Unreleased
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-587 webhcat controller map task takes too much memory (thejas via traviscrawford)
+
   HCAT-594 updated documentation, default.xml, job submission fix (thejas via gates)
 
   HCAT-588 Webhcat server wouldn't start as it is unable to write templeton.log (thejas via
traviscrawford)

Modified: incubator/hcatalog/branches/branch-0.5/src/docs/src/documentation/content/xdocs/configuration.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.5/src/docs/src/documentation/content/xdocs/configuration.xml?rev=1434845&r1=1434844&r2=1434845&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.5/src/docs/src/documentation/content/xdocs/configuration.xml
(original)
+++ incubator/hcatalog/branches/branch-0.5/src/docs/src/documentation/content/xdocs/configuration.xml
Thu Jan 17 18:22:35 2013
@@ -188,6 +188,12 @@ hive.metastore.sasl.enabled=false</code>
   </tr>
 
   <tr>
+    <td><strong>templeton.controller.mr.child.opts</strong></td>
+    <td><code>-server -Xmx256m -Djava.net.preferIPv4Stack=true</code></td>
+    <td>Java options to be passed to templeton controller map task.</td>
+  </tr>
+
+  <tr>
     <td><strong>templeton.exec.envs</strong></td>
     <td><code>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</code></td>
     <td>The environment variables passed through to exec.</td>

Modified: incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/config/webhcat-default.xml?rev=1434845&r1=1434844&r2=1434845&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/config/webhcat-default.xml
(original)
+++ incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/config/webhcat-default.xml
Thu Jan 17 18:22:35 2013
@@ -145,6 +145,21 @@
   </property>
 
   <property>
+    <name>templeton.controller.mr.child.opts</name>
+    <value>-server -Xmx256m -Djava.net.preferIPv4Stack=true</value>
+    <description>Java options to be passed to templeton controller map task.
+        The default value of mapreduce child "-Xmx" (heap memory limit)
+        might be close to what is allowed for a map task.
+        Even if templeton  controller map task does not need much 
+        memory, the jvm (with -server option?)
+        allocates the max memory when it starts. This along with the 
+        memory used by pig/hive client it starts can end up exceeding
+        the max memory configured to be allowed for a map task
+        Use this option to set -Xmx to lower value
+    </description>
+  </property>
+
+  <property>
     <name>templeton.exec.envs</name>
     <value>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</value>
     <description>The environment variables passed through to exec.</description>

Modified: incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java?rev=1434845&r1=1434844&r2=1434845&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
(original)
+++ incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
Thu Jan 17 18:22:35 2013
@@ -21,6 +21,7 @@ package org.apache.hcatalog.templeton;
 import java.io.File;
 import java.net.URL;
 import java.util.Map;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -95,6 +96,9 @@ public class AppConfig extends Configura
     public static final String TEMPLETON_JAR_NAME  = "templeton.jar";
     public static final String OVERRIDE_JARS_NAME  = "templeton.override.jars";
     public static final String OVERRIDE_JARS_ENABLED = "templeton.override.enabled";
+    public static final String TEMPLETON_CONTROLLER_MR_CHILD_OPTS 
+        = "templeton.controller.mr.child.opts";
+    
     public static final String KERBEROS_SECRET     = "templeton.kerberos.secret";
     public static final String KERBEROS_PRINCIPAL  = "templeton.kerberos.principal";
     public static final String KERBEROS_KEYTAB     = "templeton.kerberos.keytab";
@@ -103,12 +107,16 @@ public class AppConfig extends Configura
         = "templeton.callback.retry.interval";
     public static final String CALLBACK_RETRY_NAME
         = "templeton.callback.retry.attempts";
+    
+    //Hadoop property names (set by templeton logic)
     public static final String HADOOP_END_INTERVAL_NAME = "job.end.retry.interval";
     public static final String HADOOP_END_RETRY_NAME    = "job.end.retry.attempts";
     public static final String HADOOP_END_URL_NAME      = "job.end.notification.url";
     public static final String HADOOP_SPECULATIVE_NAME
         = "mapred.map.tasks.speculative.execution";
+    public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts";
 
+    
     private static final Log LOG = LogFactory.getLog(AppConfig.class);
 
     public AppConfig() {
@@ -177,7 +185,12 @@ public class AppConfig extends Configura
     public String kerberosSecret()   { return get(KERBEROS_SECRET); }
     public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); }
     public String kerberosKeytab()   { return get(KERBEROS_KEYTAB); }
+    public String controllerMRChildOpts() { 
+        return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS); 
+    }
+    
 
+    
     public String[] overrideJars() {
         if (getBoolean(OVERRIDE_JARS_ENABLED, true))
             return getStrings(OVERRIDE_JARS_NAME);

Modified: incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java?rev=1434845&r1=1434844&r2=1434845&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
(original)
+++ incubator/hcatalog/branches/branch-0.5/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/LauncherDelegator.java
Thu Jan 17 18:22:35 2013
@@ -116,6 +116,7 @@ public class LauncherDelegator extends T
         // Hadoop vars
         addDef(args, "user.name", runAs);
         addDef(args, AppConfig.HADOOP_SPECULATIVE_NAME, "false");
+        addDef(args, AppConfig.HADOOP_CHILD_JAVA_OPTS, appConf.controllerMRChildOpts());
 
         // Internal vars
         addDef(args, TempletonControllerJob.STATUSDIR_NAME, statusdir);



Mime
View raw message