incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1329506 - in /incubator/bigtop/trunk/bigtop-deploy/puppet: manifests/cluster.pp modules/hadoop/manifests/init.pp modules/hadoop/templates/core-site.xml modules/hadoop/templates/hdfs-site.xml
Date Mon, 23 Apr 2012 23:16:49 GMT
Author: rvs
Date: Mon Apr 23 23:16:49 2012
New Revision: 1329506

URL: http://svn.apache.org/viewvc?rev=1329506&view=rev
Log:
BIGTOP-539. update puppet code for BIGTOP-534

Modified:
    incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
    incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
    incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
    incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml

Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp?rev=1329506&r1=1329505&r2=1329506&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp Mon Apr 23 23:16:49 2012
@@ -22,12 +22,12 @@ class hadoop_cluster_node {
 
   $hadoop_ha = $standby_head_node ? {
     ""      => disabled,
-    default => enabled,
+    default => extlookup("hadoop_ha", "manual"),
   }
 
   $hadoop_namenode_host        = $hadoop_ha ? {
-    "enabled" => [ $hadoop_head_node, $standby_head_node ],
-    default   => $hadoop_head_node,
+    "disabled" => $hadoop_head_node,
+    default    => [ $hadoop_head_node, $standby_head_node ],
   }
   $hadoop_namenode_port        = extlookup("hadoop_namenode_port", "17020")
   $hadoop_namenode_thrift_port = extlookup("hadoop_namenode_thrift_port", "10090")
@@ -37,8 +37,8 @@ class hadoop_cluster_node {
   # $hadoop_dfs_datanode_plugins="org.apache.hadoop.thriftfs.DatanodePlugin"
   $hadoop_ha_nameservice_id    = extlookup("hadoop_ha_nameservice_id", "ha-nn-uri")
   $hadoop_namenode_uri   = $hadoop_ha ? {
-    "enabled" => "hdfs://${hadoop_ha_nameservice_id}:8020",
-    default   => "hdfs://$hadoop_namenode_host:$hadoop_namenode_port",
+    "disabled" => "hdfs://$hadoop_namenode_host:$hadoop_namenode_port",
+    default    => "hdfs://${hadoop_ha_nameservice_id}:8020",
   }
 
   $hadoop_rm_host        = $hadoop_head_node
@@ -56,6 +56,7 @@ class hadoop_cluster_node {
   $hadoop_jobtracker_thrift_port     = extlookup("hadoop_jobtracker_thrift_port", "9290")
   $hadoop_mapred_jobtracker_plugins  = extlookup("hadoop_mapred_jobtracker_plugins", "")
   $hadoop_mapred_tasktracker_plugins = extlookup("hadoop_mapred_tasktracker_plugins", "")
+  $hadoop_ha_zookeeper_quorum        = "${hadoop_head_node}:2181"
   # $hadoop_mapred_jobtracker_plugins="org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin"
   # $hadoop_mapred_tasktracker_plugins="org.apache.hadoop.mapred.TaskTrackerCmonInst"
 
@@ -134,9 +135,10 @@ class hadoop_head_node inherits hadoop_c
         # thrift_port => $hadoop_namenode_thrift_port,
         auth => $hadoop_security_authentication,
         ha   => $hadoop_ha,
+        zk   => $hadoop_ha_zookeeper_quorum,
   }
 
-  if ($hadoop_ha != "enabled") {
+  if ($hadoop_ha == "disabled") {
     hadoop::secondarynamenode { "secondary namenode":
           namenode_host => $hadoop_namenode_host,
           namenode_port => $hadoop_namenode_port,
@@ -198,7 +200,7 @@ class hadoop_head_node inherits hadoop_c
   }
 }
 
-class standby_head_node inherits hadoop_worker_node {
+class standby_head_node inherits hadoop_cluster_node {
   hadoop::namenode { "namenode":
         host => $hadoop_namenode_host,
         port => $hadoop_namenode_port,
@@ -206,6 +208,7 @@ class standby_head_node inherits hadoop_
         # thrift_port => $hadoop_namenode_thrift_port,
         auth => $hadoop_security_authentication,
         ha   => $hadoop_ha,
+        zk   => $hadoop_ha_zookeeper_quorum,
   }
 }
 

Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1329506&r1=1329505&r2=1329506&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Mon Apr 23
23:16:49 2012
@@ -75,11 +75,11 @@ class hadoop {
   }
 
   class common-hdfs inherits common {
-    if ($auth == "kerberos" and $ha == "enabled") {
+    if ($auth == "kerberos" and $ha != "disabled") {
       fail("High-availability secure clusters are not currently supported")
     }
 
-    if ($ha == 'enabled') {
+    if ($ha != 'disabled') {
       $nameservice_id = extlookup("hadoop_ha_nameservice_id", "ha-nn-uri")
     }
 
@@ -126,7 +126,7 @@ class hadoop {
     $hadoop_datanode_port           = $port
     $hadoop_security_authentication = $auth
 
-    if ($ha == 'enabled') {
+    if ($ha != 'disabled') {
       # Needed by hdfs-site.xml
       $sshfence_keydir  = "/usr/lib/hadoop/.ssh"
       $sshfence_keypath = "$sshfence_keydir/id_sshfence"
@@ -234,7 +234,7 @@ class hadoop {
     Exec <| title == "activate nn1" |>  -> Exec["HDFS init $title"]
   }
 
-  define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple",
$dirs = ["/tmp/nn"], $ha = 'disabled') {
+  define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple",
$dirs = ["/tmp/nn"], $ha = 'disabled', $zk = '') {
 
     $first_namenode = inline_template("<%= host.to_a[0] %>")
     $hadoop_namenode_host = $host
@@ -242,7 +242,7 @@ class hadoop {
     $hadoop_namenode_thrift_port = $thrift_port
     $hadoop_security_authentication = $auth
 
-    if ($ha == 'enabled') {
+    if ($ha != 'disabled') {
       $sshfence_user      = extlookup("hadoop_ha_sshfence_user",      "hdfs") 
       $sshfence_user_home = extlookup("hadoop_ha_sshfence_user_home", "/var/lib/hadoop-hdfs")
       $sshfence_keydir    = "$sshfence_user_home/.ssh"
@@ -318,24 +318,50 @@ class hadoop {
     Exec <| tag == "namenode-format" |>         -> Service["hadoop-hdfs-namenode"]
     Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-namenode"]
 
+    if ($ha == "auto") {
+      package { "hadoop-hdfs-zkfc":
+        ensure => latest,
+        require => Package["jdk"],
+      }
+
+      service { "hadoop-hdfs-zkfc":
+        ensure => running,
+        hasstatus => true,
+        subscribe => [Package["hadoop-hdfs-zkfc"], File["/etc/hadoop/conf/core-site.xml"],
File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+        require => [Package["hadoop-hdfs-zkfc"]],
+      } 
+      Service <| title == "hadoop-hdfs-zkfc" |> -> Service <| title == "hadoop-hdfs-namenode"
|>
+    }
+
     if ($::fqdn == $first_namenode) {
       exec { "namenode format":
         user => "hdfs",
-        command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log
2>&1'",
+        command => "/bin/bash -c 'yes Y | hdfs namenode -format >> /var/lib/hadoop-hdfs/nn.format.log
2>&1'",
         creates => "${namenode_data_dirs[0]}/current/VERSION",
-        require => [ Package["hadoop-hdfs-namenode"], File[$dirs] ],
+        require => [ Package["hadoop-hdfs-namenode"], File[$dirs], File["/etc/hadoop/conf/hdfs-site.xml"]
],
         tag     => "namenode-format",
       } 
-      
-      if ($ha == "enabled") {
-        exec { "activate nn1":
-          command => "/usr/bin/hdfs haadmin -transitionToActive nn1",
-          user    => "hdfs",
-          unless  => "/usr/bin/test $(/usr/bin/hdfs haadmin -getServiceState nn1) = active",
-          require => Service["hadoop-hdfs-namenode"],
+
+      if ($ha != "disabled") {
+        if ($ha == "auto") {
+          exec { "namenode zk format":
+            user => "hdfs",
+            command => "/bin/bash -c 'yes N | hdfs zkfc -formatZK >> /var/lib/hadoop-hdfs/zk.format.log
2>&1 || :'",
+            require => [ Package["hadoop-hdfs-zkfc"], File["/etc/hadoop/conf/hdfs-site.xml"]
],
+            tag     => "namenode-format",
+          }
+          Service <| title == "zookeeper-server" |> -> Exec <| title == "namenode
zk format" |>
+          Exec <| title == "namenode zk format" |>  -> Service <| title == "hadoop-hdfs-zkfc"
|>
+        } else {
+          exec { "activate nn1": 
+            command => "/usr/bin/hdfs haadmin -transitionToActive nn1",
+            user    => "hdfs",
+            unless  => "/usr/bin/test $(/usr/bin/hdfs haadmin -getServiceState nn1) =
active",
+            require => Service["hadoop-hdfs-namenode"],
+          }
         }
       }
-    } elsif ($ha == "enabled") {
+    } elsif ($ha != "disabled") {
       hadoop::namedir_copy { $namenode_data_dirs: 
         source       => $first_namenode,
         ssh_identity => $sshfence_keypath,

Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml?rev=1329506&r1=1329505&r2=1329506&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml Mon
Apr 23 23:16:49 2012
@@ -22,7 +22,7 @@
   <property>
     <!-- URI of NN. Fully qualified. No IP.-->
     <name>fs.defaultFS</name>
-<% if ha == "enabled" -%>
+<% if ha != "disabled" -%>
     <value>hdfs://<%= nameservice_id %></value>
 <% else -%>
     <value>hdfs://<%= namenode_hosts[0] %>:<%= hadoop_namenode_port%></value>

Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml?rev=1329506&r1=1329505&r2=1329506&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml Mon
Apr 23 23:16:49 2012
@@ -19,8 +19,22 @@
 <% namenode_hosts = hadoop_namenode_host.to_a -%>
 <configuration>
 
-<% if ha == "enabled" -%>
+<% if ha != "disabled" -%>
   <!-- HA -->
+
+<% if ha == "auto" -%>
+  <property>
+    <name>dfs.ha.automatic-failover.enabled</name>
+    <value>true</value>
+  </property>
+<%   if has_variable?("zk") -%>
+  <property>
+    <name>ha.zookeeper.quorum</name>
+    <value><%= zk %></value>
+  </property>
+<%   end -%>
+<%   end -%>
+
   <property> 
     <name>dfs.federation.nameservices</name>
     <value><%= nameservice_id %></value>



Mime
View raw message