incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1208881 - in /incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet: manifests/ modules/hadoop/manifests/ modules/hadoop/templates/
Date Wed, 30 Nov 2011 23:31:53 GMT
Author: rvs
Date: Wed Nov 30 23:31:47 2011
New Revision: 1208881

URL: http://svn.apache.org/viewvc?rev=1208881&view=rev
Log:
BIGTOP-273. Need to update puppet recipies for Hadoop 0.23 deployment

Added:
    incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
Modified:
    incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
    incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
    incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop
    incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
    incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
    incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp Wed Nov
30 23:31:47 2011
@@ -20,6 +20,12 @@ class hadoop_cluster_node {
   # $hadoop_dfs_namenode_plugins="org.apache.hadoop.thriftfs.NamenodePlugin"
   # $hadoop_dfs_datanode_plugins="org.apache.hadoop.thriftfs.DatanodePlugin"
 
+  $hadoop_rm_host="$hadoop_head_node"
+  $hadoop_rt_port="8025"
+  $hadoop_rm_port="8040"
+  $hadoop_sc_port="8030"
+  $hadoop_rt_thrift_port="9290"
+
   $hadoop_jobtracker_host="$hadoop_head_node"
   $hadoop_jobtracker_port="8021"
   $hadoop_jobtracker_thrift_port="9290"
@@ -35,7 +41,8 @@ class hadoop_cluster_node {
 
   $namenode_data_dirs = ["/mnt/namenode"]
   $hdfs_data_dirs = ["/mnt/hdfs"]
-  $mapred_data_dirs = ["/mnt/scratch"]
+  $mapred_data_dirs = ["/mnt/mapred"]
+  $yarn_data_dirs = ["/mnt/yarn"]
 
   if ($hadoop_security_authentication == "kerberos") {
     $kerberos_domain = "compute-1.internal"
@@ -60,12 +67,11 @@ class hadoop_worker_node inherits hadoop
         auth => $hadoop_security_authentication,
   }
 
-  hadoop::tasktracker { "tasktracker":
-        namenode_host => $hadoop_namenode_host,
-        namenode_port => $hadoop_namenode_port,
-        jobtracker_host => $hadoop_jobtracker_host,
-        jobtracker_port => $hadoop_jobtracker_port,
-        dirs => $mapred_data_dirs,
+  hadoop::nodemanager { "nodemanager":
+        rm_host => $hadoop_rm_host,
+        rm_port => $hadoop_rm_port,
+        rt_port => $hadoop_rt_port,
+        dirs => $yarn_data_dirs,
         auth => $hadoop_security_authentication,
   }
 
@@ -74,6 +80,15 @@ class hadoop_worker_node inherits hadoop
         zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
         kerberos_realm => $kerberos_realm, 
   }
+
+  hadoop::mapred-app { "mapred-app":
+        namenode_host => $hadoop_namenode_host,
+        namenode_port => $hadoop_namenode_port,
+        jobtracker_host => $hadoop_jobtracker_host,
+        jobtracker_port => $hadoop_jobtracker_port,
+        auth => $hadoop_security_authentication,
+        dirs => $mapred_data_dirs,
+  }
 }
 
 class hadoop_head_node inherits hadoop_cluster_node {
@@ -83,9 +98,8 @@ class hadoop_head_node inherits hadoop_c
   }
 
   hadoop::namenode { "namenode":
+        host => $hadoop_namenode_host,
         port => $hadoop_namenode_port,
-        jobtracker_host => $hadoop_jobtracker_host,
-        jobtracker_port => $hadoop_jobtracker_port,
         dirs => $namenode_data_dirs,
         # thrift_port => $hadoop_namenode_thrift_port,
         auth => $hadoop_security_authentication,
@@ -97,12 +111,11 @@ class hadoop_head_node inherits hadoop_c
         auth => $hadoop_security_authentication,
   }
 
-  hadoop::jobtracker { "jobtracker":
-        namenode_host => $hadoop_namenode_host,
-        namenode_port => $hadoop_namenode_port,
-        host => $hadoop_jobtracker_host,
-        port => $hadoop_jobtracker_port,
-        dirs => $mapred_data_dirs,
+  hadoop::resourcemanager { "resourcemanager":
+        host => $hadoop_rm_host,
+        port => $hadoop_rm_port,
+        rt_port => $hadoop_rt_port,
+        sc_port => $hadoop_sc_port,
         # thrift_port => $hadoop_jobtracker_thrift_port,
         auth => $hadoop_security_authentication,
   }
@@ -122,7 +135,7 @@ class hadoop_head_node inherits hadoop_c
         ensemble => $hadoop_zookeeper_ensemble,
   }
 
-  hadoop::create_hdfs_dirs { [ "/mapred", "/tmp", "/system", "/user", "/hbase", "/benchmarks",
"/user/jenkins", "/user/hive" ]:
+  hadoop::create_hdfs_dirs { [ "/mapred", "/tmp", "/system", "/user", "/hbase", "/benchmarks",
"/user/jenkins", "/user/hive", "/user/root", "/user/history" ]:
     hdfs_dirs_meta => { "/tmp"          => { perm => "777", user => "hdfs"  
},
                         "/mapred"       => { perm => "755", user => "mapred" },
                         "/system"       => { perm => "755", user => "hdfs"   },
@@ -130,19 +143,20 @@ class hadoop_head_node inherits hadoop_c
                         "/hbase"        => { perm => "755", user => "hbase"  },
                         "/benchmarks"   => { perm => "777", user => "hdfs"   },
                         "/user/jenkins" => { perm => "777", user => "jenkins"},
+                        "/user/history" => { perm => "777", user => "mapred" },
+                        "/user/root"    => { perm => "777", user => "root"   },
                         "/user/hive"    => { perm => "777", user => "hive"   } },
   }
 }
 
 class hadoop_gateway_node inherits hadoop_head_node {
-  # hadoop::client { "gateway":
-  #   namenode_host => $hadoop_namenode_host,
-  #   namenode_port => $hadoop_namenode_port,
-  #   jobtracker_host => $hadoop_jobtracker_host,
-  #   jobtracker_port => $hadoop_jobtracker_port,
-  #   # auth => $hadoop_security_authentication,
-  # }
-
+  hadoop::client { "hadoop client":
+    namenode_host => $hadoop_namenode_host,
+    namenode_port => $hadoop_namenode_port,
+    jobtracker_host => $hadoop_jobtracker_host,
+    jobtracker_port => $hadoop_jobtracker_port,
+    # auth => $hadoop_security_authentication,
+  }
   mahout::client { "mahout client":
   }
   hadoop-pig::client { "pig client":

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
(original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
Wed Nov 30 23:31:47 2011
@@ -21,46 +21,63 @@ class hadoop {
    */
   class common {
     file {
-      "/etc/hadoop/conf/core-site.xml":
-        content => template('hadoop/core-site.xml'),
+      "/etc/hadoop/conf/hadoop-env.sh":
+        content => template('hadoop/hadoop-env.sh'),
         require => [Package["hadoop"]],
     }
 
     file {
-      "/etc/hadoop/conf/mapred-site.xml":
-        content => template('hadoop/mapred-site.xml'),
+      "/etc/default/hadoop":
+        content => template('hadoop/hadoop'),
         require => [Package["hadoop"]],
     }
 
+    package { "hadoop":
+      ensure => latest,
+      require => Package["jdk"],
+    }
+
+    #FIXME: package { "hadoop-native":
+    #  ensure => latest,
+    #  require => [Package["hadoop"]],
+    #}
+  }
+
+  class common-yarn inherits common {
     file {
-      "/etc/hadoop/conf/hdfs-site.xml":
-        content => template('hadoop/hdfs-site.xml'),
+      "/etc/yarn/conf/yarn-site.xml":
+        content => template('hadoop/yarn-site.xml'),
         require => [Package["hadoop"]],
     }
+  }
 
+  class common-hdfs inherits common {
     file {
-      "/etc/hadoop/conf/hadoop-env.sh":
-        content => template('hadoop/hadoop-env.sh'),
+      "/etc/hadoop/conf/core-site.xml":
+        content => template('hadoop/core-site.xml'),
         require => [Package["hadoop"]],
     }
 
     file {
-      "/etc/default/hadoop":
-        content => template('hadoop/hadoop'),
+      "/etc/hadoop/conf/hdfs-site.xml":
+        content => template('hadoop/hdfs-site.xml'),
         require => [Package["hadoop"]],
     }
+  }
 
-    package { "hadoop":
-      ensure => latest,
+  class common-mapred-app inherits common-hdfs {
+    file {
+      "/etc/hadoop/conf/mapred-site.xml":
+        content => template('hadoop/mapred-site.xml'),
+        require => [Package["hadoop"]],
     }
 
-    package { "hadoop-native":
-      ensure => latest,
+    file { "/etc/hadoop/conf/taskcontroller.cfg":
+      content => template('hadoop/taskcontroller.cfg'), 
       require => [Package["hadoop"]],
     }
   }
 
-
   define datanode ($namenode_host, $namenode_port, $port = "50075", $auth = "simple", $dirs
= ["/tmp/data"]) {
 
     $hadoop_namenode_host = $namenode_host
@@ -68,7 +85,7 @@ class hadoop {
     $hadoop_datanode_port = $port
     $hadoop_security_authentication = $auth
 
-    include common
+    include common-hdfs
 
     package { "hadoop-datanode":
       ensure => latest,
@@ -76,10 +93,10 @@ class hadoop {
     }
 
     if ($hadoop_security_authentication == "kerberos") {
-      package { "hadoop-sbin":
-        ensure => latest,
-        require => [Package["hadoop"]],
-      }
+      #FIXME: package { "hadoop-sbin":
+      #  ensure => latest,
+      #  require => [Package["hadoop"]],
+      #}
     }
 
     service { "hadoop-datanode":
@@ -108,16 +125,14 @@ class hadoop {
     }
   }
 
-  define namenode ($jobtracker_host, $jobtracker_port, $host = $fqdn , $port = "8020", $thrift_port=
"10090", $auth = "simple", $dirs = ["/tmp/nn"]) {
+  define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple",
$dirs = ["/tmp/nn"]) {
 
     $hadoop_namenode_host = $host
     $hadoop_namenode_port = $port
     $hadoop_namenode_thrift_port = $thrift_port
-    $hadoop_jobtracker_host = $jobtracker_host
-    $hadoop_jobtracker_port = $jobtracker_port
     $hadoop_security_authentication = $auth
 
-    include common
+    include common-hdfs
 
     package { "hadoop-namenode":
       ensure => latest,
@@ -127,14 +142,14 @@ class hadoop {
     service { "hadoop-namenode":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-namenode"], File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
+      subscribe => [Package["hadoop-namenode"], File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
       require => [Package["hadoop-namenode"], Exec["namenode format"]],
     } 
 
     exec { "namenode format":
       user => "hdfs",
       command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log
2>&1'",
-      creates => inline_template("<%= hadoop_storage_locations.split(';')[0] %>/namenode/image"),
+      creates => inline_template("<%= hadoop_storage_locations.split(';')[0] %>/namenode/current/VERSION"),
       require => [Package["hadoop-namenode"]],
     } <- file { $dirs:
       ensure => directory,
@@ -145,88 +160,91 @@ class hadoop {
     }
   }
 
+  define secondarynamenode ($namenode_host, $namenode_port, $port = "50090", $auth = "simple")
{
 
-  define jobtracker ($namenode_host, $namenode_port, $host = $fqdn, $port = "8021", $thrift_port
= "9290", $auth = "simple", $dirs = ["/tmp/mr"]) {
-
-    $hadoop_namenode_host = $namenode_host
-      $hadoop_namenode_port = $namenode_port
-      $hadoop_jobtracker_thrift_port = $thrift_port
-      $hadoop_jobtracker_host = $host
-      $hadoop_jobtracker_port = $port
-      $hadoop_security_authentication = $auth
+    $hadoop_secondarynamenode_port = $port
+    $hadoop_security_authentication = $auth
 
-      include common
+    include common-hdfs
 
-      package { "hadoop-jobtracker":
-        ensure => latest,
-               require => Package["jdk"],
-      }
+    package { "hadoop-secondarynamenode":
+      ensure => latest,
+      require => Package["jdk"],
+    }
 
-    service { "hadoop-jobtracker":
+    service { "hadoop-secondarynamenode":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-jobtracker"], File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/mapred-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [ Package["hadoop-jobtracker"] ]
-    } <- file { $dirs:
-      ensure => directory,
-      owner => mapred,
-      group => mapred,
-      mode => 755,
-      require => [Package["hadoop"]],
+      subscribe => [Package["hadoop-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+      require => [Package["hadoop-secondarynamenode"]],
     }
   }
 
 
-  define tasktracker ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port,
$auth = "simple", $dirs = ["/tmp/mr"]){
+  define resourcemanager ($host = $fqdn, $port = "8040", $rt_port = "8025", $sc_port = "8030",
$thrift_port = "9290", $auth = "simple") {
+    $hadoop_rm_host = $host
+    $hadoop_rm_port = $port
+    $hadoop_rt_port = $rt_port
+    $hadoop_sc_port = $sc_port
+    $hadoop_security_authentication = $auth
 
-    $hadoop_namenode_host = $namenode_host
-      $hadoop_namenode_port = $namenode_port
-      $hadoop_jobtracker_host = $jobtracker_host
-      $hadoop_jobtracker_port = $jobtracker_port
-      $hadoop_security_authentication = $auth
+    include common-yarn
 
-      include common
+    package { "hadoop-resourcemanager":
+      ensure => latest,
+      require => Package["jdk"],
+    }
 
-      package { "hadoop-tasktracker":
-        ensure => latest,
-        require => Package["jdk"],
-      }
- 
-    file { "/etc/hadoop/conf/taskcontroller.cfg":
-      content => template('hadoop/taskcontroller.cfg'), 
+    service { "hadoop-resourcemanager":
+      ensure => running,
+      hasstatus => true,
+      subscribe => [Package["hadoop-resourcemanager"], File["/etc/hadoop/conf/hadoop-env.sh"],
File["/etc/yarn/conf/yarn-site.xml"]],
+      require => [ Package["hadoop-resourcemanager"] ]
     }
+  }
+
+
+  define nodemanager ($rm_host, $rm_port, $rt_port, $auth = "simple", $dirs = ["/tmp/yarn"]){
+    $hadoop_rm_host = $rm_host
+    $hadoop_rm_port = $rm_port
+    $hadoop_rt_port = $rt_port
 
-    service { "hadoop-tasktracker":
+    include common-yarn
+
+    package { "hadoop-nodemanager":
+      ensure => latest,
+      require => Package["jdk"],
+    }
+ 
+    service { "hadoop-nodemanager":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-tasktracker"], File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/mapred-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [ Package["hadoop-tasktracker"], File["/etc/hadoop/conf/taskcontroller.cfg"]
],
+      subscribe => [Package["hadoop-nodemanager"], File["/etc/hadoop/conf/hadoop-env.sh"],
File["/etc/yarn/conf/yarn-site.xml"]],
+      require => [ Package["hadoop-nodemanager"] ],
     } <- file { $dirs:
       ensure => directory,
-      owner => mapred,
-      group => mapred,
+      owner => yarn,
+      group => yarn,
       mode => 755,
       require => [Package["hadoop"]],
     }
   }
 
-
-  define secondarynamenode ($namenode_host, $namenode_port, $port = "50090", $auth = "simple")
{
-
-    $hadoop_secondarynamenode_port = $port
+  define mapred-app ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port,
$auth = "simple", $dirs = ["/tmp/mr"]){
+    $hadoop_namenode_host = $namenode_host
+    $hadoop_namenode_port = $namenode_port
+    $hadoop_jobtracker_host = $jobtracker_host
+    $hadoop_jobtracker_port = $jobtracker_port
     $hadoop_security_authentication = $auth
-      include common
 
-      package { "hadoop-secondarynamenode":
-        ensure => latest,
-        require => Package["jdk"],
-      }
+    include common-mapred-app
 
-    service { "hadoop-secondarynamenode":
-      ensure => running,
-      hasstatus => true,
-      subscribe => [Package["hadoop-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [Package["hadoop-secondarynamenode"]],
+    file { $dirs:
+      ensure => directory,
+      owner => mapred,
+      group => mapred,
+      mode => 755,
+      require => [Package["hadoop"]],
     }
   }
 
@@ -237,10 +255,10 @@ class hadoop {
       $hadoop_jobtracker_port = $jobtracker_port
       $hadoop_security_authentication = $auth
 
-      include common
+      include common-mapred-app
   
-      package { ["hadoop-doc", "hadoop-source", "hadoop-debuginfo", 
-                 "hadoop-fuse", "hadoop-libhdfs", "hadoop-pipes"]:
+      # FIXME: "hadoop-source", "hadoop-fuse", "hadoop-pipes"
+      package { ["hadoop-doc", "hadoop-debuginfo", "hadoop-libhdfs"]:
         ensure => latest,
         require => [Package["jdk"], Package["hadoop"]],  
       }

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop
(original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hadoop
Wed Nov 30 23:31:47 2011
@@ -12,9 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
+export HADOOP_PREFIX=/usr/lib/hadoop
+export HADOOP_LIBEXEC_DIR=/usr/libexec
 export HADOOP_NAMENODE_USER=hdfs
 export HADOOP_SECONDARYNAMENODE_USER=hdfs
 export HADOOP_DATANODE_USER=hdfs
@@ -25,4 +24,13 @@ export HADOOP_SECURE_DN_LOG_DIR=$HADOOP_
 <% end %>
 export HADOOP_JOBTRACKER_USER=mapred
 export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hadoop
+export HADOOP_IDENT_STRING=hdfs
+export HADOOP_PID_DIR=/var/run/hadoop
+export HADOOP_LOG_DIR=/var/log/hadoop
+export HADOOP_CONF_DIR=/etc/hadoop/conf
+export YARN_CONF_DIR=/etc/yarn/conf
+export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HADOOP_PREFIX}/*:${HADOOP_PREFIX}/lib/*"
+
+# Pointless definitions making yarn happy
+export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
+export HADOOP_HDFS_HOME=${HADOOP_PREFIX}

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
(original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
Wed Nov 30 23:31:47 2011
@@ -116,6 +116,12 @@
     <value><% hadoop_storage_locations.split(";").each do |storage_location| %><%=
storage_location%>/namenode,<% end %></value>
   </property>
 
+  <property>
+    <name>dfs.permissions.superusergroup</name>
+    <value>hadoop</value>
+    <description>The name of the group of super-users.</description>
+  </property>
+
   <!-- Enable Hue plugins -->
 <% if has_variable?("hadoop_dfs_namenode_plugins") %>
   <property>

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
(original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
Wed Nov 30 23:31:47 2011
@@ -49,7 +49,7 @@
   
   <!-- TaskController settings -->
   <property>
-    <name>mapred.task.tracker.task-controller</name>
+    <name>mapreduce.tasktracker.taskcontroller</name>
     <value>org.apache.hadoop.mapred.LinuxTaskController</value>
   </property>
   <property>
@@ -60,7 +60,7 @@
 
 <% if has_variable?("mapred_acls_enabled") %>
   <property>
-    <name>mapred.acls.enabled</name>
+    <name>mapreduce.cluster.acls.enabled</name>
     <value><%= mapred_acls_enabled %></value>
   </property>
 <% end %>
@@ -68,7 +68,7 @@
 <!-- specify JobTracker TaskScheduler -->
 <% if has_variable?("hadoop_jobtracker_taskscheduler") %>
   <property>
-    <name>mapred.jobtracker.taskScheduler</name>
+    <name>mapreduce.jobtracker.taskscheduler</name>
     <value><%= hadoop_jobtracker_taskscheduler %></value>
   </property>
 <% end %>
@@ -95,19 +95,34 @@
 <% end %>
 
   <property>
-    <name>mapred.job.tracker</name>
+    <name>mapreduce.framework.name</name>
+    <value>yarn</value>
+  </property>
+
+  <property>
+    <name>mapreduce.admin.user.env</name>
+    <value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.address</name>
     <value><%= hadoop_jobtracker_host %>:<%= hadoop_jobtracker_port%></value>
   </property>
 
   <property>
-    <name>mapred.local.dir</name>
-    <value><% hadoop_storage_locations.split(";").each do |storage_location| %><%=
storage_location%>/scratch,<% end %></value>
+    <name>mapreduce.cluster.local.dir</name>
+    <value><% hadoop_storage_locations.split(";").each do |storage_location| %><%=
storage_location%>/mapred,<% end %></value>
     <final>true</final>
   </property>
 
+  <!-- property>
+    <name>mapreduce.cluster.temp.dir</name>
+    <value>/mapred/system</value>
+  </property -->
+
   <property>
-    <name>mapred.child.java.opts</name>
-    <value>-Xmx1024m</value>
+    <name>mapreduce.jobtracker.system.dir</name>
+    <value>/mapred/system</value>
   </property>
 
   <property>
@@ -116,8 +131,13 @@
   </property>
 
   <property>
-    <name>mapred.system.dir</name>
-    <value>/mapred/system</value>
+    <name>yarn.app.mapreduce.am.staging-dir</name>
+    <value>/user</value>
+  </property>
+
+  <property>
+    <name>mapred.child.java.opts</name>
+    <value>-Xmx1024m</value>
   </property>
 
 <% if has_variable?("hadoop_config_mapred_child_ulimit") %>

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg?rev=1208881&r1=1208880&r2=1208881&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
(original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
Wed Nov 30 23:31:47 2011
@@ -1,4 +1,4 @@
-mapred.local.dir=<% hadoop_storage_locations.split(";").each do |storage_location| %><%=
storage_location%>/scratch,<% end %>
+mapred.local.dir=<% hadoop_storage_locations.split(";").each do |storage_location| %><%=
storage_location%>/mapred,<% end %>
 hadoop.log.dir=/var/log/hadoop/
 mapreduce.tasktracker.group=mapred
 min.user.id=0

Added: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml?rev=1208881&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
(added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
Wed Nov 30 23:31:47 2011
@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+  <property>
+    <name>yarn.resourcemanager.resource-tracker.address</name>
+    <value><%= hadoop_rm_host %>:<%= hadoop_rt_port %></value>
+  </property>
+
+  <property>
+    <name>yarn.resourcemanager.address</name>
+    <value><%= hadoop_rm_host %>:<%= hadoop_rm_port %></value>
+  </property>
+
+  <property>
+    <name>yarn.resourcemanager.scheduler.address</name>
+    <value><%= hadoop_rm_host %>:<%= hadoop_sc_port %></value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services</name>
+    <value>mapreduce.shuffle</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.local-dirs</name>
+    <value><% hadoop_storage_locations.split(";").each do |storage_location| %><%=
storage_location%>/yarn,<% end %></value>
+    <final>true</final>
+  </property>
+
+</configuration>



Mime
View raw message