hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ra...@apache.org
Subject svn commit: r1190044 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/packages/hadoop-setup-conf.sh src/main/packages/templates/conf/mapred-site.xml
Date Thu, 27 Oct 2011 21:31:37 GMT
Author: ramya
Date: Thu Oct 27 21:31:37 2011
New Revision: 1190044

URL: http://svn.apache.org/viewvc?rev=1190044&view=rev
Log:
HADOOP-7728. Enable task memory management to be configurable in hadoop config setup script.
(ramya)

Modified:
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
    hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1190044&r1=1190043&r2=1190044&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Thu
Oct 27 21:31:37 2011
@@ -68,6 +68,9 @@ Release 0.23.0 - Unreleased
 
   IMPROVEMENTS
 
+    HADOOP-7728. Enable task memory management to be configurable in hadoop
+    config setup script. (ramya)
+
     HADOOP-7655. Provide a small validation script that smoke tests the installed
     cluster. (Arpit Gupta via mattf)
 

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh?rev=1190044&r1=1190043&r2=1190044&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
Thu Oct 27 21:31:37 2011
@@ -55,6 +55,16 @@ usage: $0 <parameters>
      --dfs-support-append=false|true                                 Enable append
      --hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts'    Setup proxy users for
hadoop
      --hbase-user=hbase                                              User which hbase is
running as. Defaults to hbase
+     --mapreduce-cluster-mapmemory-mb=memory                         Virtual memory of a
map slot for the MR framework. Defaults to -1
+     --mapreduce-cluster-reducememory-mb=memory                      Virtual memory, of a
reduce slot for the MR framework. Defaults to -1
+     --mapreduce-jobtracker-maxmapmemory-mb=memory                   Maximum virtual memory
of a single map task. Defaults to -1
+                                                                     This value should be
set to (mapreduce.cluster.mapmemory.mb * mapreduce.tasktracker.map.tasks.maximum)
+     --mapreduce-jobtracker-maxreducememory-mb=memory                Maximum virtual memory
of a single reduce task. Defaults to -1
+                                                                     This value should be
set to (mapreduce.cluster.reducememory.mb * mapreduce.tasktracker.reduce.tasks.maximum)
+     --mapreduce-map-memory-mb=memory                                Virtual memory of a
single map slot for a job. Defaults to -1
+                                                                     This value should be
<= mapred.cluster.max.map.memory.mb
+     --mapreduce-reduce-memory-mb=memory                             Virtual memory, of a
single reduce slot for a job. Defaults to -1
+                                                                     This value should be
<= mapred.cluster.max.reduce.memory.mb
   "
   exit 1
 }
@@ -199,6 +209,12 @@ OPTS=$(getopt \
   -l 'hadoop-proxy-users:' \
   -l 'dfs-support-append:' \
   -l 'hbase-user:' \
+  -l 'mapreduce-cluster-mapmemory-mb:' \
+  -l 'mapreduce-cluster-reducememory-mb:' \
+  -l 'mapreduce-jobtracker-maxmapmemory-mb:' \
+  -l 'mapreduce-jobtracker-maxreducememory-mb:' \
+  -l 'mapreduce-map-memory-mb:' \
+  -l 'mapreduce-reduce-memory-mb:' \
   -o 'h' \
   -- "$@") 
   
@@ -334,6 +350,30 @@ while true ; do
       HBASE_USER=$2; shift 2
       AUTOMATED=1
       ;;
+    --mapreduce-cluster-mapmemory-mb)
+      MAPREDUCE_CLUSTER_MAPMEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-cluster-reducememory-mb)
+      MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-jobtracker-maxmapmemory-mb)
+      MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-jobtracker-maxreducememory-mb)
+      MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-map-memory-mb)
+      MAPREDUCE_MAP_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-reduce-memory-mb)
+      MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
     --)
       shift ; break
       ;;
@@ -365,6 +405,12 @@ HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
 DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
 DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
 HBASE_USER=${HBASE_USER:-hbase}
+MAPREDUCE_CLUSTER_MAPMEMORY_MB=${MAPREDUCE_CLUSTER_MAPMEMORY_MB:--1}
+MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB:--1}
+MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB:--1}
+MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB:--1}
+MAPREDUCE_MAP_MEMORY_MB=${MAPREDUCE_MAP_MEMORY_MB:--1}
+MAPREDUCE_REDUCE_MEMORY_MB=${MAPREDUCE_REDUCE_MEMORY_MB:--1}
 KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
 HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
 MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}

Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml?rev=1190044&r1=1190043&r2=1190044&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
Thu Oct 27 21:31:37 2011
@@ -272,4 +272,34 @@
     <name>mapred.jobtracker.retirejob.interval</name>
     <value>0</value>
   </property>
+
+  <property>
+    <name>mapreduce.cluster.mapmemory.mb</name>
+    <value>${MAPREDUCE_CLUSTER_MAPMEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.cluster.reducememory.mb</name>
+    <value>${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.maxmapmemory.mb</name>
+    <value>${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.maxreducememory.mb</name>
+    <value>${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.map.memory.mb</name>
+    <value>${MAPREDUCE_MAP_MEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.reduce.memory.mb</name>
+    <value>${MAPREDUCE_REDUCE_MEMORY_MB}</value>
+  </property>
 </configuration>



Mime
View raw message