incubator-ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject svn commit: r1362314 - in /incubator/ambari/trunk: ./ hmc/db/ hmc/php/frontend/ hmc/php/puppet/ hmc/php/upgrade/dbupgrade/ hmc/php/util/ hmc/puppet/modules/hdp-hadoop/manifests/ hmc/puppet/modules/hdp-hadoop/templates/
Date Mon, 16 Jul 2012 23:47:49 GMT
Author: hitesh
Date: Mon Jul 16 23:47:48 2012
New Revision: 1362314

URL: http://svn.apache.org/viewvc?rev=1362314&view=rev
Log:
AMBARI-616. Enable support for configuration of heapsize for TaskTracker (Contributed by Hitesh)

Modified:
    incubator/ambari/trunk/CHANGES.txt
    incubator/ambari/trunk/hmc/db/schema.dump
    incubator/ambari/trunk/hmc/php/frontend/configUtils.php
    incubator/ambari/trunk/hmc/php/puppet/DBReader.php
    incubator/ambari/trunk/hmc/php/upgrade/dbupgrade/upgrade_1.0.dump
    incubator/ambari/trunk/hmc/php/util/suggestProperties.php
    incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/manifests/params.pp
    incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb

Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1362314&r1=1362313&r2=1362314&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Mon Jul 16 23:47:48 2012
@@ -12,6 +12,9 @@ Trunk (unreleased changes)
 
   NEW FEATURES
 
+    AMBARI-616. Enable support for configuration of heapsize for TaskTracker
+    (hitesh)
+
     AMBARI-636. Support for Hadoop Security (front-end changes)
     (Jaimin Jetly via yusaku)
 

Modified: incubator/ambari/trunk/hmc/db/schema.dump
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/db/schema.dump?rev=1362314&r1=1362313&r2=1362314&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/db/schema.dump (original)
+++ incubator/ambari/trunk/hmc/db/schema.dump Mon Jul 16 23:47:48 2012
@@ -323,6 +323,9 @@ INSERT OR REPLACE INTO "ConfigProperties
 -- maps to jt_heapsize in gscluster in MB.
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description,
service_name, display_type, display_attributes ) VALUES ( "jtnode_heapsize", "1024", "JobTracker
maximum Java heap size", "Maximum Java heap size for JobTracker in MB (Java option -Xmx)",
"MAPREDUCE" , "TEXT", '{ "reconfigurable": true, "unit":"MB" }' );
 
+-- maps to ttnode_heapsize in gscluster in MB.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description,
service_name, display_type, display_attributes ) VALUES ( "ttnode_heapsize", "1024", "TaskTracker
maximum Java heap size", "Maximum Java heap size for TaskTracker in MB (Java option -Xmx)",
"MAPREDUCE" , "TEXT", '{ "reconfigurable": true, "unit":"MB" }' );
+
 -- maps to mapred_map_tasks_max in gscluster.
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description,
service_name, display_type, display_attributes ) VALUES ( "mapred_map_tasks_max", "4", "Number
of Map slots per node", "Number of slots that Map tasks that run simultaneously can occupy
on a TaskTracker", "MAPREDUCE" , "TEXT", '{ "reconfigurable": true, "unit":"int" }' );
 

Modified: incubator/ambari/trunk/hmc/php/frontend/configUtils.php
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/php/frontend/configUtils.php?rev=1362314&r1=1362313&r2=1362314&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/php/frontend/configUtils.php (original)
+++ incubator/ambari/trunk/hmc/php/frontend/configUtils.php Mon Jul 16 23:47:48 2012
@@ -78,7 +78,6 @@ function basicNumericCheck($val, $negati
 function validateConfigs($svcConfigs) {
   $REQUIRED_FIELD_MESSAGE = 'This is required.  Please specify.';
   $errors = array();
-//  foreach ($configs as $svc => $svcConfigs) {
     foreach ($svcConfigs as $key => $val) {
       $val = trim($val);
       if ($key == "dfs_name_dir") {
@@ -288,6 +287,11 @@ function validateConfigs($svcConfigs) {
         if ($check["error"] != "") {
           $errors[$key] = $check;
         }
+      } else if ($key == "ttnode_heapsize") {
+        $check = basicNumericCheck($val, FALSE);
+        if ($check["error"] != "") {
+          $errors[$key] = $check;
+        }
       } else if ($key == "mapred_map_tasks_max") {
         $check = basicNumericCheck($val);
         if ($check["error"] != "") {
@@ -507,52 +511,7 @@ function validateConfigs($svcConfigs) {
           $errors[$key] = array ( "error" => $REQUIRED_FIELD_MESSAGE);
         }
       }
-
-      /*
-      hive_mysql_host|
-      hive_database_name|
-      hive_metastore_user_name|
-      hive_metastore_user_passwd|
-      java32_home|
-      java64_home|
-      jdk_location|
-      dfs_support_append|true
-      dfs_webhdfs_enabled|false
-      scheduler_name|org.apache.hadoop.mapred.CapacityTaskScheduler
-      nagios_web_login|nagiosadmin
-      nagios_web_password|admin
-      hadoop_heapsize|1024
-      namenode_heapsize|1024
-      namenode_opt_newsize|200
-      datanode_du_reserved|1073741824
-      dtnode_heapsize|1024
-      jtnode_opt_newsize|200
-      jtnode_opt_maxnewsize|200
-      jtnode_heapsize|1024
-      mapred_map_tasks_max|4
-      mapred_red_tasks_max|2
-      mapred_cluster_map_mem_mb|-1
-      mapred_cluster_red_mem_mb|-1
-      mapred_cluster_max_map_mem_mb|-1
-      mapred_cluster_max_red_mem_mb|-1
-      mapred_job_map_mem_mb|-1
-      mapred_job_red_mem_mb|-1
-      mapred_child_java_opts_sz|768
-      io_sort_mb|200
-      io_sort_spill_percent|0.9
-      mapreduce_userlog_retainhours|24
-      maxtasks_per_job|-1
-      dfs_datanode_failed_volume_tolerated|0
-      tickTime|2000
-      initLimit|10
-      syncLimit|5
-      clientPort|2181
-      hbase_master_heapsize|1024
-      hbase_regionserver_heapsize|1024
-      */
-
     }
-//  }
   $result = 0;
   if (!empty($errors)) {
     $result = -1;

Modified: incubator/ambari/trunk/hmc/php/puppet/DBReader.php
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/php/puppet/DBReader.php?rev=1362314&r1=1362313&r2=1362314&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/php/puppet/DBReader.php (original)
+++ incubator/ambari/trunk/hmc/php/puppet/DBReader.php Mon Jul 16 23:47:48 2012
@@ -228,6 +228,7 @@ class DBReader {
      "jtnode_opt_newsize" => "m",
      "jtnode_opt_maxnewsize" => "m",
      "jtnode_heapsize" => "m",
+     "ttnode_heapsize" => "m",
      "mapred_child_java_opts_sz" => "m",
      "io_sort_mb" => "m",
      "hbase_master_heapsize" => "m",

Modified: incubator/ambari/trunk/hmc/php/upgrade/dbupgrade/upgrade_1.0.dump
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/php/upgrade/dbupgrade/upgrade_1.0.dump?rev=1362314&r1=1362313&r2=1362314&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/php/upgrade/dbupgrade/upgrade_1.0.dump (original)
+++ incubator/ambari/trunk/hmc/php/upgrade/dbupgrade/upgrade_1.0.dump Mon Jul 16 23:47:48
2012
@@ -106,6 +106,9 @@ INSERT OR REPLACE INTO "ConfigProperties
 -- maps to jt_heapsize in gscluster in MB.
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description,
service_name, display_type, display_attributes ) VALUES ( "jtnode_heapsize", "1024", "JobTracker
maximum Java heap size", "Maximum Java heap size for JobTracker in MB (Java option -Xmx)",
"MAPREDUCE" , "TEXT", '{ "reconfigurable": true, "unit":"MB" }' );
 
+-- maps to ttnode_heapsize in gscluster in MB.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description,
service_name, display_type, display_attributes ) VALUES ( "ttnode_heapsize", "1024", "TaskTracker
maximum Java heap size", "Maximum Java heap size for TaskTracker in MB (Java option -Xmx)",
"MAPREDUCE" , "TEXT", '{ "reconfigurable": true, "unit":"MB" }' );
+
 -- maps to mapred_map_tasks_max in gscluster.
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description,
service_name, display_type, display_attributes ) VALUES ( "mapred_map_tasks_max", "4", "Number
of Map slots per node", "Number of slots that Map tasks that run simultaneously can occupy
on a TaskTracker", "MAPREDUCE" , "TEXT", '{ "reconfigurable": true, "unit":"int" }' );
 

Modified: incubator/ambari/trunk/hmc/php/util/suggestProperties.php
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/php/util/suggestProperties.php?rev=1362314&r1=1362313&r2=1362314&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/php/util/suggestProperties.php (original)
+++ incubator/ambari/trunk/hmc/php/util/suggestProperties.php Mon Jul 16 23:47:48 2012
@@ -273,11 +273,12 @@ class SuggestProperties {
 
     // TODO fix - this should be based on heap size divided by max task
     // limit on the host
-    $heapSize = $this->allocateHeapSizeForDaemon("TASKTRACKER", $hostRoles,
-        $hostInfoMap, $allHostsToComponents, TRUE);
+    $heapSize = $this->allocateHeapSizeWithMax("TASKTRACKER", $hostRoles,
+        $hostInfoMap, $allHostsToComponents, TRUE, 2048);
     $heapSizeWithMax = $this->allocateHeapSizeWithMax("TASKTRACKER", $hostRoles,
         $hostInfoMap, $allHostsToComponents, TRUE, 3072);
     $this->logger->log_info("Maxed Heap Size for MR Child opts ".$heapSizeWithMax);
+    $result["configs"]["ttnode_heapsize"] = $heapSize;
     $result["configs"]["mapred_child_java_opts_sz"] = $heapSizeWithMax;
 
     if (array_key_exists("HBASE", $services)) {
@@ -427,6 +428,7 @@ class SuggestProperties {
         "jtnode_heapsize" => array ( "role" => "JOBTRACKER", "32bit" => FALSE),
         "dtnode_heapsize" => array ( "role" => "DATANODE", "32bit" => TRUE),
         "hadoop_heapsize" => array ( "role" => "DATANODE", "32bit" => TRUE),
+        "ttnode_heapsize" => array ( "role" => "TASKTRACKER", "32bit" => TRUE),
         "mapred_child_java_opts_sz" => array ( "role" => "TASKTRACKER", "32bit" =>
TRUE)
       );
 

Modified: incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/manifests/params.pp?rev=1362314&r1=1362313&r2=1362314&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/manifests/params.pp (original)
+++ incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/manifests/params.pp Mon Jul 16 23:47:48
2012
@@ -35,6 +35,7 @@ class hdp-hadoop::params(
   ### hadoop-env
   
   $dtnode_heapsize = hdp_default("hadoop/hadoop-env/dtnode_heapsize","1024m")
+  $ttnode_heapsize = hdp_default("hadoop/hadoop-env/ttnode_heapsize","1024m")
 
   $hadoop_heapsize = hdp_default("hadoop/hadoop-env/hadoop_heapsize","1024m")
 

Modified: incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb?rev=1362314&r1=1362313&r2=1362314&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb (original)
+++ incubator/ambari/trunk/hmc/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb Mon Jul
16 23:47:48 2012
@@ -43,7 +43,7 @@ export HADOOP_OPTS="-Djava.net.preferIPv
 export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=<%=scope.function_hdp_template_var("hadoop_logdirprefix")%>/log/$USER/hs_err_pid%p.log
-XX:NewSize=<%=scope.function_hdp_template_var("namenode_opt_newsize")%> -XX:MaxNewSize=<%=scope.function_hdp_template_var("namenode_opt_maxnewsize")%>
-Xloggc:<%=scope.function_hdp_template_var("hadoop_logdirprefix")%>/log/$USER/gc.log-`date
+'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps
-Xms<%=scope.function_hdp_template_var("namenode_heapsize")%> -Xmx<%=scope.function_hdp_template_var("namenode_heapsize")%>
-Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
 HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=<%=scope.function_hdp_template_var("hadoop_logdirprefix")%>/log/$USER/hs_err_pid%p.log
-XX:NewSize=<%=scope.function_hdp_template_var("jtnode_opt_newsize")%> -XX:MaxNewSize=<%=scope.function_hdp_template_var("jtnode_opt_maxnewsize")%>
-Xloggc:<%=scope.function_hdp_template_var("hadoop_logdirprefix")%>/log/$USER/gc.log-`date
+'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps
-Xmx<%=scope.function_hdp_template_var("jtnode_heapsize")%> -Dhadoop.security.logger=INFO,DRFAS
-Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
 
-HADOOP_TASKTRACKER_OPTS="-server -Xmx1024m -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console
${HADOOP_TASKTRACKER_OPTS}"
+HADOOP_TASKTRACKER_OPTS="-server -Xmx<%=scope.function_hdp_template_var("ttnode_heapsize")%>
-Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
 HADOOP_DATANODE_OPTS="-Xmx<%=scope.function_hdp_template_var("dtnode_heapsize")%> -Dhadoop.security.logger=ERROR,DRFAS
${HADOOP_DATANODE_OPTS}"
 HADOOP_BALANCER_OPTS="-server -Xmx<%=scope.function_hdp_template_var("hadoop_heapsize")%>
${HADOOP_BALANCER_OPTS}"
 



Mime
View raw message