incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1326670 [2/47] - in /incubator/bigtop/trunk: ./ bigtop-deploy/puppet/manifests/ bigtop-deploy/puppet/modules/hadoop-hbase/manifests/ bigtop-deploy/puppet/modules/hadoop-hbase/templates/ bigtop-deploy/puppet/modules/hadoop-oozie/manifests/ ...
Date Mon, 16 Apr 2012 16:10:32 GMT
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Mon Apr 16 16:10:22 2012
@@ -22,12 +22,15 @@ class hadoop {
    
   class kerberos {
     require kerberos::client
-    
+
     kerberos::host_keytab { "hdfs":
       princs => [ "host", "hdfs" ],
+      spnego => true,
     }
    
     kerberos::host_keytab { [ "yarn", "mapred" ]:
+      tag    => "mapreduce",
+      spnego => true,
     }
   }
 
@@ -37,84 +40,172 @@ class hadoop {
     }
 
     file {
-      "/etc/hadoop/conf/core-site.xml":
-        content => template('hadoop/core-site.xml'),
+      "/etc/hadoop/conf/hadoop-env.sh":
+        content => template('hadoop/hadoop-env.sh'),
         require => [Package["hadoop"]],
     }
 
-    file {
-      "/etc/hadoop/conf/mapred-site.xml":
-        content => template('hadoop/mapred-site.xml'),
-        require => [Package["hadoop"]],
+    package { "hadoop":
+      ensure => latest,
+      require => Package["jdk"],
     }
 
+    #FIXME: package { "hadoop-native":
+    #  ensure => latest,
+    #  require => [Package["hadoop"]],
+    #}
+  }
+
+  class common-yarn inherits common {
+    package { "hadoop-yarn":
+      ensure => latest,
+      require => [Package["jdk"], Package["hadoop"]],
+    }
+ 
     file {
-      "/etc/hadoop/conf/hdfs-site.xml":
-        content => template('hadoop/hdfs-site.xml'),
+      "/etc/hadoop/conf/yarn-site.xml":
+        content => template('hadoop/yarn-site.xml'),
         require => [Package["hadoop"]],
     }
 
+    file { "/etc/hadoop/conf/container-executor.cfg":
+      content => template('hadoop/container-executor.cfg'), 
+      require => [Package["hadoop"]],
+    }
+  }
+
+  class common-hdfs inherits common {
+    if ($auth == "kerberos" and $ha == "enabled") {
+      fail("High-availability secure clusters are not currently supported")
+    }
+
+    if ($ha == 'enabled') {
+      $nameservice_id = extlookup("hadoop_ha_nameservice_id", "ha-nn-uri")
+    }
+
+    package { "hadoop-hdfs":
+      ensure => latest,
+      require => [Package["jdk"], Package["hadoop"]],
+    }
+ 
     file {
-      "/etc/hadoop/conf/hadoop-env.sh":
-        content => template('hadoop/hadoop-env.sh'),
+      "/etc/hadoop/conf/core-site.xml":
+        content => template('hadoop/core-site.xml'),
         require => [Package["hadoop"]],
     }
 
     file {
-      "/etc/default/hadoop":
-        content => template('hadoop/hadoop'),
+      "/etc/hadoop/conf/hdfs-site.xml":
+        content => template('hadoop/hdfs-site.xml'),
         require => [Package["hadoop"]],
     }
+  }
 
-    package { "hadoop":
+  class common-mapred-app inherits common-hdfs {
+    package { "hadoop-mapreduce":
       ensure => latest,
+      require => [Package["jdk"], Package["hadoop"]],
     }
 
-    package { "hadoop-native":
-      ensure => latest,
+    file {
+      "/etc/hadoop/conf/mapred-site.xml":
+        content => template('hadoop/mapred-site.xml'),
+        require => [Package["hadoop"]],
+    }
+
+    file { "/etc/hadoop/conf/taskcontroller.cfg":
+      content => template('hadoop/taskcontroller.cfg'), 
       require => [Package["hadoop"]],
     }
   }
 
+  define datanode ($namenode_host, $namenode_port, $port = "50075", $auth = "simple", $dirs = ["/tmp/data"], $ha = 'disabled') {
 
-  define datanode ($namenode_host, $namenode_port, $port = "50075", $auth = "simple", $dirs = ["/tmp/data"]) {
-
-    $hadoop_namenode_host = $namenode_host
-    $hadoop_namenode_port = $namenode_port
-    $hadoop_datanode_port = $port
+    $hadoop_namenode_host           = $namenode_host
+    $hadoop_namenode_port           = $namenode_port
+    $hadoop_datanode_port           = $port
     $hadoop_security_authentication = $auth
 
-    include common
+    if ($ha == 'enabled') {
+      # Needed by hdfs-site.xml
+      $sshfence_keydir  = "/usr/lib/hadoop/.ssh"
+      $sshfence_keypath = "$sshfence_keydir/id_sshfence"
+      $sshfence_user    = extlookup("hadoop_ha_sshfence_user",    "hdfs") 
+      $shared_edits_dir = extlookup("hadoop_ha_shared_edits_dir", "/hdfs_shared")
+    }
+
+    include common-hdfs
 
-    package { "hadoop-datanode":
+    package { "hadoop-hdfs-datanode":
       ensure => latest,
       require => Package["jdk"],
     }
 
-    if ($hadoop_security_authentication == "kerberos") {
-      package { "hadoop-sbin":
-        ensure => latest,
-        require => [Package["hadoop"]],
-      }
+    file {
+      "/etc/default/hadoop-hdfs-datanode":
+        content => template('hadoop/hadoop-hdfs'),
+        require => [Package["hadoop-hdfs-datanode"]],
     }
 
-    service { "hadoop-datanode":
+    service { "hadoop-hdfs-datanode":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-datanode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [ Package["hadoop-datanode"], File[$dirs] ],
+      subscribe => [Package["hadoop-hdfs-datanode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+      require => [ Package["hadoop-hdfs-datanode"], File[$dirs] ],
     }
-    Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-datanode"]
+    Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-datanode"]
 
     file { $dirs:
       ensure => directory,
       owner => hdfs,
       group => hdfs,
       mode => 755,
-      require => [Package["hadoop"]],
+      require => [ Package["hadoop-hdfs"] ],
     }
   }
 
+  define httpfs ($namenode_host, $namenode_port, $port = "14000", $auth = "simple", $secret = "hadoop httpfs secret") {
+
+    $hadoop_namenode_host = $namenode_host
+    $hadoop_namenode_port = $namenode_port
+    $hadoop_httpfs_port = $port
+    $hadoop_security_authentication = $auth
+
+    if ($auth == "kerberos") {
+      kerberos::host_keytab { "httpfs":
+        spnego => true,
+      }
+    }
+
+    package { "hadoop-httpfs":
+      ensure => latest,
+      require => Package["jdk"],
+    }
+
+    file { "/etc/hadoop-httpfs/conf/httpfs-site.xml":
+      content => template('hadoop/httpfs-site.xml'),
+      require => [Package["hadoop-httpfs"]],
+    }
+
+    file { "/etc/hadoop-httpfs/conf/httpfs-env.sh":
+      content => template('hadoop/httpfs-env.sh'),
+      require => [Package["hadoop-httpfs"]],
+    }
+
+    file { "/etc/hadoop-httpfs/conf/httpfs-signature.secret":
+      content => inline_template("<%= secret %>"),
+      require => [Package["hadoop-httpfs"]],
+    }
+
+    service { "hadoop-httpfs":
+      ensure => running,
+      hasstatus => true,
+      subscribe => [Package["hadoop-httpfs"], File["/etc/hadoop-httpfs/conf/httpfs-site.xml"], File["/etc/hadoop-httpfs/conf/httpfs-env.sh"], File["/etc/hadoop-httpfs/conf/httpfs-signature.secret"]],
+      require => [ Package["hadoop-httpfs"] ],
+    }
+    Kerberos::Host_keytab <| title == "httpfs" |> -> Service["hadoop-httpfs"]
+  }
+
   class kinit {
     include hadoop::kerberos
 
@@ -138,140 +229,293 @@ class hadoop {
       user => "hdfs",
       command => "/bin/bash -c 'hadoop fs -mkdir $title && hadoop fs -chmod $perm $title && hadoop fs -chown $user $title'",
       unless => "/bin/bash -c 'hadoop fs -ls $name >/dev/null 2>&1'",
-      require => [ Service["hadoop-namenode"], Exec["namenode format"] ],
+      require => Service["hadoop-hdfs-namenode"],
     }
+    Exec <| title == "activate nn1" |>  -> Exec["HDFS init $title"]
   }
 
-  define namenode ($jobtracker_host, $jobtracker_port, $host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"]) {
+  define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"], $ha = 'disabled') {
 
+    $first_namenode = inline_template("<%= host.to_a[0] %>")
     $hadoop_namenode_host = $host
     $hadoop_namenode_port = $port
     $hadoop_namenode_thrift_port = $thrift_port
-    $hadoop_jobtracker_host = $jobtracker_host
-    $hadoop_jobtracker_port = $jobtracker_port
     $hadoop_security_authentication = $auth
 
-    include common
+    if ($ha == 'enabled') {
+      $sshfence_user      = extlookup("hadoop_ha_sshfence_user",      "hdfs") 
+      $sshfence_user_home = extlookup("hadoop_ha_sshfence_user_home", "/var/lib/hadoop-hdfs")
+      $sshfence_keydir    = "$sshfence_user_home/.ssh"
+      $sshfence_keypath   = "$sshfence_keydir/id_sshfence"
+      $sshfence_privkey   = extlookup("hadoop_ha_sshfence_privkey",   "$extlookup_datadir/hadoop/id_sshfence")
+      $sshfence_pubkey    = extlookup("hadoop_ha_sshfence_pubkey",    "$extlookup_datadir/hadoop/id_sshfence.pub")
+      $shared_edits_dir   = extlookup("hadoop_ha_shared_edits_dir",   "/hdfs_shared")
+      $nfs_server         = extlookup("hadoop_ha_nfs_server",         "")
+      $nfs_path           = extlookup("hadoop_ha_nfs_path",           "")
+
+      file { $sshfence_keydir:
+        ensure  => directory,
+        owner   => 'hdfs',
+        group   => 'hdfs',
+        mode    => '0700',
+        require => Package["hadoop-hdfs"],
+      }
+
+      file { $sshfence_keypath:
+        source  => $sshfence_privkey,
+        owner   => 'hdfs',
+        group   => 'hdfs',
+        mode    => '0600',
+        before  => Service["hadoop-hdfs-namenode"],
+        require => File[$sshfence_keydir],
+      }
+
+      file { "$sshfence_keydir/authorized_keys":
+        source  => $sshfence_pubkey,
+        owner   => 'hdfs',
+        group   => 'hdfs',
+        mode    => '0600',
+        before  => Service["hadoop-hdfs-namenode"],
+        require => File[$sshfence_keydir],
+      }
+
+      file { $shared_edits_dir:
+        ensure => directory,
+      }
+
+      if ($nfs_server) {
+        if (!$nfs_path) {
+          fail("No nfs share specified for shared edits dir")
+        }
+
+        require nfs::client
+
+        mount { $shared_edits_dir:
+          ensure  => "mounted",
+          atboot  => true,
+          device  => "${nfs_server}:${nfs_path}",
+          fstype  => "nfs",
+          options => "tcp,soft,timeo=10,intr,rsize=32768,wsize=32768",
+          require => File[$shared_edits_dir],
+          before  => Service["hadoop-hdfs-namenode"],
+        }
+      }
+    }
+
+    include common-hdfs
 
-    package { "hadoop-namenode":
+    package { "hadoop-hdfs-namenode":
       ensure => latest,
       require => Package["jdk"],
     }
 
-    service { "hadoop-namenode":
+    service { "hadoop-hdfs-namenode":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [Package["hadoop-namenode"], Exec["namenode format"]],
+      subscribe => [Package["hadoop-hdfs-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+      require => [Package["hadoop-hdfs-namenode"]],
     } 
-    Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-namenode"]
+    Exec <| tag == "namenode-format" |>         -> Service["hadoop-hdfs-namenode"]
+    Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-namenode"]
 
-    exec { "namenode format":
-      user => "hdfs",
-      command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
-      creates => "${namenode_data_dirs[0]}/image",
-      require => [ Package["hadoop-namenode"], File[$dirs] ],
-    } 
+    if ($::fqdn == $first_namenode) {
+      exec { "namenode format":
+        user => "hdfs",
+        command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
+        creates => "${namenode_data_dirs[0]}/current/VERSION",
+        require => [ Package["hadoop-hdfs-namenode"], File[$dirs] ],
+        tag     => "namenode-format",
+      } 
+      
+      if ($ha == "enabled") {
+        exec { "activate nn1":
+          command => "/usr/bin/hdfs haadmin -transitionToActive nn1",
+          user    => "hdfs",
+          unless  => "/usr/bin/test $(/usr/bin/hdfs haadmin -getServiceState nn1) = active",
+          require => Service["hadoop-hdfs-namenode"],
+        }
+      }
+    } elsif ($ha == "enabled") {
+      hadoop::namedir_copy { $namenode_data_dirs: 
+        source       => $first_namenode,
+        ssh_identity => $sshfence_keypath,
+        require      => File[$sshfence_keypath],
+      }
+    }
+
+    file {
+      "/etc/default/hadoop-hdfs-namenode":
+        content => template('hadoop/hadoop-hdfs'),
+        require => [Package["hadoop-hdfs-namenode"]],
+    }
     
     file { $dirs:
       ensure => directory,
       owner => hdfs,
       group => hdfs,
       mode => 700,
-      require => [Package["hadoop"]], 
+      require => [Package["hadoop-hdfs"]], 
     }
   }
 
+  define namedir_copy ($source, $ssh_identity) {
+    exec { "copy namedir $title from first namenode":
+      command => "/usr/bin/rsync -avz -e '/usr/bin/ssh -oStrictHostKeyChecking=no -i $ssh_identity' '${source}:$title/' '$title/'",
+      user    => "hdfs",
+      tag     => "namenode-format",
+      creates => "$title/current/VERSION",
+    }
+  }
+      
+  define secondarynamenode ($namenode_host, $namenode_port, $port = "50090", $auth = "simple") {
 
-  define jobtracker ($namenode_host, $namenode_port, $host = $fqdn, $port = "8021", $thrift_port = "9290", $auth = "simple", $dirs = ["/tmp/mr"]) {
+    $hadoop_secondarynamenode_port = $port
+    $hadoop_security_authentication = $auth
 
-    $hadoop_namenode_host = $namenode_host
-      $hadoop_namenode_port = $namenode_port
-      $hadoop_jobtracker_thrift_port = $thrift_port
-      $hadoop_jobtracker_host = $host
-      $hadoop_jobtracker_port = $port
-      $hadoop_security_authentication = $auth
+    include common-hdfs
 
-      include common
+    package { "hadoop-hdfs-secondarynamenode":
+      ensure => latest,
+      require => Package["jdk"],
+    }
 
-      package { "hadoop-jobtracker":
-        ensure => latest,
-               require => Package["jdk"],
-      }
+    file {
+      "/etc/default/hadoop-hdfs-secondarynamenode":
+        content => template('hadoop/hadoop-hdfs'),
+        require => [Package["hadoop-hdfs-secondarynamenode"]],
+    }
 
-    service { "hadoop-jobtracker":
+    service { "hadoop-hdfs-secondarynamenode":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-jobtracker"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/mapred-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [ Package["hadoop-jobtracker"], File[$dirs] ],
+      subscribe => [Package["hadoop-hdfs-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+      require => [Package["hadoop-hdfs-secondarynamenode"]],
     }
-   
-    file { $dirs:
-      ensure => directory,
-      owner => mapred,
-      group => mapred,
-      mode => 755,
-      require => [Package["hadoop"]],
+    Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-secondarynamenode"]
+  }
+
+
+  define resourcemanager ($host = $fqdn, $port = "8040", $rt_port = "8025", $sc_port = "8030", $thrift_port = "9290", $auth = "simple") {
+    $hadoop_rm_host = $host
+    $hadoop_rm_port = $port
+    $hadoop_rt_port = $rt_port
+    $hadoop_sc_port = $sc_port
+    $hadoop_security_authentication = $auth
+
+    include common-yarn
+
+    package { "hadoop-yarn-resourcemanager":
+      ensure => latest,
+      require => Package["jdk"],
+    }
+
+    service { "hadoop-yarn-resourcemanager":
+      ensure => running,
+      hasstatus => true,
+      subscribe => [Package["hadoop-yarn-resourcemanager"], File["/etc/hadoop/conf/hadoop-env.sh"], 
+                    File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+      require => [ Package["hadoop-yarn-resourcemanager"] ],
     }
-    Kerberos::Host_keytab <| title == "mapred" |> -> Service["hadoop-jobtracker"]
+    Kerberos::Host_keytab <| tag == "mapreduce" |> -> Service["hadoop-yarn-resourcemanager"]
   }
 
+  define proxyserver ($host = $fqdn, $port = "8088", $auth = "simple") {
+    $hadoop_ps_host = $host
+    $hadoop_ps_port = $port
+    $hadoop_security_authentication = $auth
 
-  define tasktracker ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port, $auth = "simple", $dirs = ["/tmp/mr"]){
+    include common-yarn
 
-    $hadoop_namenode_host = $namenode_host
-      $hadoop_namenode_port = $namenode_port
-      $hadoop_jobtracker_host = $jobtracker_host
-      $hadoop_jobtracker_port = $jobtracker_port
-      $hadoop_security_authentication = $auth
+    package { "hadoop-yarn-proxyserver":
+      ensure => latest,
+      require => Package["jdk"],
+    }
 
-      include common
+    service { "hadoop-yarn-proxyserver":
+      ensure => running,
+      hasstatus => true,
+      subscribe => [Package["hadoop-yarn-proxyserver"], File["/etc/hadoop/conf/hadoop-env.sh"], 
+                    File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+      require => [ Package["hadoop-yarn-proxyserver"] ],
+    }
+    Kerberos::Host_keytab <| tag == "mapreduce" |> -> Service["hadoop-yarn-proxyserver"]
+  }
 
-      package { "hadoop-tasktracker":
-        ensure => latest,
-        require => Package["jdk"],
-      }
- 
-    file { "/etc/hadoop/conf/taskcontroller.cfg":
-      content => template('hadoop/taskcontroller.cfg'), 
+  define historyserver ($host = $fqdn, $port = "10020", $webapp_port = "19888", $auth = "simple") {
+    $hadoop_hs_host = $host
+    $hadoop_hs_port = $port
+    $hadoop_hs_webapp_port = $app_port
+    $hadoop_security_authentication = $auth
+
+    include common-mapred-app
+
+    package { "hadoop-mapreduce-historyserver":
+      ensure => latest,
+      require => Package["jdk"],
     }
 
-    service { "hadoop-tasktracker":
+    service { "hadoop-mapreduce-historyserver":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-tasktracker"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/mapred-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [ Package["hadoop-tasktracker"], File["/etc/hadoop/conf/taskcontroller.cfg"], File[$dirs] ],
+      subscribe => [Package["hadoop-mapreduce-historyserver"], File["/etc/hadoop/conf/hadoop-env.sh"], 
+                    File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+      require => [Package["hadoop-mapreduce-historyserver"]],
+    }
+    Kerberos::Host_keytab <| tag == "mapreduce" |> -> Service["hadoop-mapreduce-historyserver"]
+  }
+
+
+  define nodemanager ($rm_host, $rm_port, $rt_port, $auth = "simple", $dirs = ["/tmp/yarn"]){
+    $hadoop_rm_host = $rm_host
+    $hadoop_rm_port = $rm_port
+    $hadoop_rt_port = $rt_port
+
+    include common-yarn
+
+    package { "hadoop-yarn-nodemanager":
+      ensure => latest,
+      require => Package["jdk"],
     }
  
+    service { "hadoop-yarn-nodemanager":
+      ensure => running,
+      hasstatus => true,
+      subscribe => [Package["hadoop-yarn-nodemanager"], File["/etc/hadoop/conf/hadoop-env.sh"], 
+                    File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+      require => [ Package["hadoop-yarn-nodemanager"], File[$dirs] ],
+    }
+    Kerberos::Host_keytab <| tag == "mapreduce" |> -> Service["hadoop-yarn-nodemanager"]
+
     file { $dirs:
       ensure => directory,
-      owner => mapred,
-      group => mapred,
+      owner => yarn,
+      group => yarn,
       mode => 755,
-      require => [Package["hadoop"]],
+      require => [Package["hadoop-yarn"]],
     }
-    Kerberos::Host_keytab <| title == "mapred" |> -> Service["hadoop-tasktracker"]
   }
 
-
-  define secondarynamenode ($namenode_host, $namenode_port, $port = "50090", $auth = "simple") {
-
-    $hadoop_secondarynamenode_port = $port
+  define mapred-app ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port, $auth = "simple", $jobhistory_host = "", $jobhistory_port="10020", $dirs = ["/tmp/mr"]){
+    $hadoop_namenode_host = $namenode_host
+    $hadoop_namenode_port = $namenode_port
+    $hadoop_jobtracker_host = $jobtracker_host
+    $hadoop_jobtracker_port = $jobtracker_port
     $hadoop_security_authentication = $auth
-      include common
 
-      package { "hadoop-secondarynamenode":
-        ensure => latest,
-        require => Package["jdk"],
-      }
+    include common-mapred-app
 
-    service { "hadoop-secondarynamenode":
-      ensure => running,
-      hasstatus => true,
-      subscribe => [Package["hadoop-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [Package["hadoop-secondarynamenode"]],
+    if ($jobhistory_host != "") {
+      $hadoop_hs_host = $jobhistory_host
+      $hadoop_hs_port = $jobhistory_port
+    }
+
+    file { $dirs:
+      ensure => directory,
+      owner => yarn,
+      group => yarn,
+      mode => 755,
+      require => [Package["hadoop-mapreduce"]],
     }
-    Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-secondarynamenode"]
   }
 
   define client ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port, $auth = "simple") {
@@ -281,12 +525,12 @@ class hadoop {
       $hadoop_jobtracker_port = $jobtracker_port
       $hadoop_security_authentication = $auth
 
-      include common
+      include common-mapred-app
   
-      package { ["hadoop-doc", "hadoop-source", "hadoop-debuginfo", 
-                 "hadoop-fuse", "hadoop-libhdfs", "hadoop-pipes"]:
+      # FIXME: "hadoop-source", "hadoop-fuse", "hadoop-pipes"
+      package { ["hadoop-doc", "hadoop-debuginfo", "hadoop-libhdfs"]:
         ensure => latest,
-        require => [Package["jdk"], Package["hadoop"]],  
+        require => [Package["jdk"], Package["hadoop"], Package["hadoop-hdfs"], Package["hadoop-mapreduce"]],  
       }
   }
 }

Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg Mon Apr 16 16:10:22 2012
@@ -0,0 +1,3 @@
+yarn.nodemanager.linux-container-executor.group=yarn
+#banned.users=foo,bar
+#min.user.id=1000

Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml Mon Apr 16 16:10:22 2012
@@ -16,14 +16,20 @@
 <!-- See the License for the specific language governing permissions and      -->
 <!-- limitations under the License.                                           -->
 
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
 <configuration>
 
   <property>
     <!-- URI of NN. Fully qualified. No IP.-->
-    <name>fs.default.name</name>
-    <value>hdfs://<%= hadoop_namenode_host %>:<%= hadoop_namenode_port%></value>
+    <name>fs.defaultFS</name>
+<% if ha == "enabled" -%>
+    <value>hdfs://<%= nameservice_id %></value>
+<% else -%>
+    <value>hdfs://<%= namenode_hosts[0] %>:<%= hadoop_namenode_port%></value>
+<% end -%>
   </property>
 
+
   <property>
     <name>hadoop.security.authentication</name>
     <value><%= hadoop_security_authentication %></value>
@@ -56,11 +62,6 @@
 <% end %>
 
 <% end %>
-  <property>
-    <name>dfs.permissions</name>
-    <value>false</value>
-  </property>
-
 <% if has_variable?("hadoop_snappy_codec") %>
   <property>
     <name>io.compression.codecs</name>

Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs (from r1325587, incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs&p1=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs Mon Apr 16 16:10:22 2012
@@ -11,18 +11,16 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
+# limitations under the License.
+export HADOOP_PID_DIR=/var/run/hadoop-hdfs
+export HADOOP_LOG_DIR=/var/log/hadoop-hdfs
 export HADOOP_NAMENODE_USER=hdfs
 export HADOOP_SECONDARYNAMENODE_USER=hdfs
 export HADOOP_DATANODE_USER=hdfs
+export HADOOP_IDENT_STRING=hdfs
+
 <% if hadoop_security_authentication == "kerberos" %>
 export HADOOP_SECURE_DN_USER=hdfs
 export HADOOP_SECURE_DN_PID_DIR=$HADOOP_PID_DIR
 export HADOOP_SECURE_DN_LOG_DIR=$HADOOP_LOG_DIR
 <% end %>
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hadoop

Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml Mon Apr 16 16:10:22 2012
@@ -16,9 +16,64 @@
 <!-- See the License for the specific language governing permissions and      -->
 <!-- limitations under the License.                                           -->
 
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
 <configuration>
 
-<% if hadoop_security_authentication == "kerberos" %>
+<% if ha == "enabled" -%>
+  <!-- HA -->
+  <property> 
+    <name>dfs.federation.nameservices</name>
+    <value><%= nameservice_id %></value>
+  </property>
+
+  <property>
+    <name>dfs.ha.namenodes.<%= nameservice_id %></name>
+    <value><%= (1..namenode_hosts.length).map { |n| "nn#{n}" }.join(",") %></value>
+  </property>
+
+<%   namenode_hosts.each_with_index do |host,idx| -%>
+  <property>
+    <name>dfs.namenode.rpc-address.<%= nameservice_id %>.nn<%= idx+1 %></name>
+    <value><%= host %>:<%= hadoop_namenode_port %></value>
+  </property>
+ 
+  <property>
+    <name>dfs.namenode.http-address.<%= nameservice_id %>.nn<%= idx+1 %></name>
+    <value><%= host %>:50070</value>
+  </property>
+
+<%   end -%>
+<%   if has_variable?("shared_edits_dir") -%>
+  <property>
+    <name>dfs.namenode.shared.edits.dir</name>
+    <value><%= shared_edits_dir %></value>
+  </property>
+
+<%   end -%>
+  <property>
+    <name>dfs.client.failover.proxy.provider.<%= nameservice_id %></name>
+    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
+  </property>
+
+<%   if has_variable?("sshfence_privkey") -%>
+  <property>
+    <name>dfs.ha.fencing.methods</name>
+    <value>sshfence(<%= sshfence_user %>)</value>
+  </property>
+
+  <property>
+    <name>dfs.ha.fencing.ssh.private-key-files</name>
+    <value><%= sshfence_keypath %></value>
+  </property>
+
+<%   else -%>
+  <property>
+    <name>dfs.ha.fencing.methods</name>
+    <value></value>
+  </property>
+
+<%   end -%>
+<% elsif hadoop_security_authentication == "kerberos" -%>
   <property>
     <name>dfs.block.access.token.enable</name>
     <value>true</value>
@@ -27,7 +82,7 @@
   <!-- NameNode security config -->
   <property>
     <name>dfs.https.address</name>
-    <value><%= hadoop_namenode_host %>:50475</value>
+    <value><%= namenode_hosts[0] %>:50475</value>
   </property>
   <property>
     <name>dfs.https.port</name>
@@ -45,15 +100,24 @@
     <name>dfs.namenode.kerberos.https.principal</name>
     <value>host/_HOST@<%= kerberos_realm %></value>
   </property>
+  <property>
+    <name>dfs.web.authentication.kerberos.keytab</name>
+    <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
+  </property>
+  <property>
+    <name>dfs.web.authentication.kerberos.principal</name>
+    <value>HTTP/_HOST@<%= kerberos_realm %></value>
+  </property>
+
   
   <!-- Secondary NameNode security config -->
   <property>
     <name>dfs.secondary.http.address</name>
-    <value><%= hadoop_namenode_host %>:0</value>
+    <value><%= namenode_hosts[0] %>:0</value>
   </property>
   <property>
     <name>dfs.secondary.https.address</name>
-    <value><%= hadoop_namenode_host %>:50495</value>
+    <value><%= namenode_hosts[0] %>:50495</value>
   </property>
   <property>
     <name>dfs.secondary.https.port</name>
@@ -97,16 +161,10 @@
     <name>dfs.datanode.kerberos.https.principal</name>
     <value>host/_HOST@<%= kerberos_realm %></value>
   </property>
-<% end %>
 
+<% end -%>
   <!-- name node -->
   <property>
-    <!-- URI of NN. Fully qualified. No IP.-->
-    <name>fs.default.name</name>
-    <value>hdfs://<%= hadoop_namenode_host %>:<%= hadoop_namenode_port%></value>
-  </property>
- 
-  <property>
     <name>dfs.data.dir</name>
     <value><%= hdfs_data_dirs.join(",") %></value>
   </property>
@@ -116,30 +174,38 @@
     <value><%= namenode_data_dirs.join(",") %></value>
   </property>
 
+  <property>
+    <name>dfs.permissions.superusergroup</name>
+    <value>hadoop</value>
+    <description>The name of the group of super-users.</description>
+  </property>
+
   <!-- Enable Hue plugins -->
-<% if has_variable?("hadoop_dfs_namenode_plugins") %>
+<% if has_variable?("hadoop_dfs_namenode_plugins") -%>
   <property>
     <name>dfs.namenode.plugins</name>
     <value><%= hadoop_dfs_namenode_plugins %></value>
     <description>Comma-separated list of namenode plug-ins to be activated.
     </description>
   </property>
-<% end %>
-<% if has_variable?("hadoop_dfs_datanode_plugins") %>
+
+<% end -%>
+<% if has_variable?("hadoop_dfs_datanode_plugins") -%>
   <property>
     <name>dfs.datanode.plugins</name>
     <value><%= hadoop_dfs_datanode_plugins %></value>
     <description>Comma-separated list of datanode plug-ins to be activated.
     </description>
   </property>
-<% end %>
-<% if has_variable?("hadoop_namenode_thrift_port") %>
+
+<% end -%>
+<% if has_variable?("hadoop_namenode_thrift_port") -%>
   <property>
     <name>dfs.thrift.address</name>
     <value>0.0.0.0:<%= hadoop_namenode_thrift_port %></value>
   </property>
-<% end %>
 
+<% end -%>
   <!-- increase the number of datanode transceivers way above the default of 256
      - this is for hbase -->
   <property>
@@ -148,18 +214,23 @@
   </property>
 
   <!-- Configurations for large cluster -->
-<% if has_variable?("hadoop_config_dfs_block_size") %>
+<% if has_variable?("hadoop_config_dfs_block_size") -%>
   <property>
     <name>dfs.block.size</name>
     <value><%= hadoop_config_dfs_block_size %></value>
   </property>
-<% end %>
 
-<% if has_variable?("hadoop_config_namenode_handler_count") %>
+<% end -%>
+<% if has_variable?("hadoop_config_namenode_handler_count") -%>
   <property>
     <name>dfs.namenode.handler.count</name>
     <value><%= hadoop_config_namenode_handler_count %></value>
   </property>
-<% end %>
+
+<% end -%>
+  <property>
+    <name>dfs.webhdfs.enabled</name>
+    <value>true</value>
+  </property>
 
 </configuration>

Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-env.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-env.sh?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-env.sh (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-env.sh Mon Apr 16 16:10:22 2012
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+#
+
+# Set httpfs specific environment variables here.
+
+# Settings for the Embedded Tomcat that runs HttpFS
+# Java System properties for HttpFS should be specified in this variable
+#
+# export CATALINA_OPTS=
+
+# HttpFS logs directory
+#
+# export HTTPFS_LOG=${HTTPFS_HOME}/logs
+
+# HttpFS temporary directory
+#
+# export HTTPFS_TEMP=${HTTPFS_HOME}/temp
+
+# The HTTP port used by HttpFS
+#
+export HTTPFS_HTTP_PORT=<%= hadoop_httpfs_port %>
+
+# The Admin port used by HttpFS
+#
+# export HTTPFS_ADMIN_PORT=`expr ${HTTPFS_HTTP_PORT} + 1`
+
+# The hostname HttpFS server runs on
+#
+# export HTTPFS_HTTP_HOSTNAME=`hostname -f`

Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml Mon Apr 16 16:10:22 2012
@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more       -->
+<!-- contributor license agreements.  See the NOTICE file distributed with    -->
+<!-- this work for additional information regarding copyright ownership.      -->
+<!-- The ASF licenses this file to You under the Apache License, Version 2.0  -->
+<!-- (the "License"); you may not use this file except in compliance with     -->
+<!-- the License.  You may obtain a copy of the License at                    -->
+<!--                                                                          -->
+<!--     http://www.apache.org/licenses/LICENSE-2.0                           -->
+<!--                                                                          -->
+<!-- Unless required by applicable law or agreed to in writing, software      -->
+<!-- distributed under the License is distributed on an "AS IS" BASIS,        -->
+<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
+<!-- See the License for the specific language governing permissions and      -->
+<!-- limitations under the License.                                           -->
+
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
+<configuration>
+
+<% if has_variable?("hadoop_core_proxyusers") -%>
+<% hadoop_core_proxyusers.sort.each do |superuser, data| -%>
+  <property>
+    <name>httpfs.proxyuser.<%= superuser %>.hosts</name>
+    <value><%= data['hosts'] %></value>
+  </property>
+  <property>
+    <name>httpfs.proxyuser.<%= superuser %>.groups</name>
+    <value><%= data['groups'] %></value>
+  </property>
+<% end -%>
+<% end -%>
+
+<% if hadoop_security_authentication == "kerberos" -%>
+  <property>
+    <name>httpfs.authentication.type</name>
+    <value>kerberos</value>
+  </property>
+  <property>
+    <name>kerberos.realm</name>
+    <value><%= kerberos_realm %></value>
+  </property>
+  <property>
+    <name>httpfs.authentication.kerberos.principal</name>
+    <value>HTTP/<%= fqdn %></value>
+  </property>
+  <property>
+    <name>httpfs.authentication.kerberos.keytab</name>
+    <value>/etc/httpfs.keytab</value>
+  </property>
+
+  <property>
+    <name>httpfs.hadoop.authentication.type</name>
+    <value>kerberos</value>
+  </property>
+  <property>
+    <name>httpfs.hadoop.authentication.kerberos.principal</name>
+    <value>httpfs/<%= fqdn %></value>
+  </property>
+  <property>
+    <name>httpfs.hadoop.authentication.kerberos.keytab</name>
+    <value>${httpfs.authentication.kerberos.keytab}</value>
+  </property>
+  <property>
+    <name>httpfs.hadoop.conf:dfs.namenode.kerberos.principal</name>
+    <value>hdfs/<%= namenode_hosts[0] %>@<%= kerberos_realm %></value>
+  </property>
+<% end -%>
+
+  <property>
+    <name>httpfs.hadoop.conf:fs.default.name</name>
+    <value>hdfs://<%= namenode_hosts[0] %>:<%= hadoop_namenode_port%></value>
+  </property>
+</configuration>

Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml Mon Apr 16 16:10:22 2012
@@ -19,6 +19,16 @@
 <configuration>
 
 <% if hadoop_security_authentication == "kerberos" %>
+  <!-- HistoryServer security configs -->
+  <property>
+    <name>mapreduce.jobhistory.principal</name>
+    <value>mapred/_HOST@<%= kerberos_realm %></value>
+  </property>
+  <property>
+    <name>mapreduce.jobhistory.keytab</name>
+    <value>/etc/mapred.keytab</value> <!-- path to the MapReduce keytab -->
+  </property>
+
   <!-- JobTracker security configs -->
   <property>
     <name>mapreduce.jobtracker.kerberos.principal</name>
@@ -49,7 +59,7 @@
   
   <!-- TaskController settings -->
   <property>
-    <name>mapred.task.tracker.task-controller</name>
+    <name>mapreduce.tasktracker.taskcontroller</name>
     <value>org.apache.hadoop.mapred.LinuxTaskController</value>
   </property>
   <property>
@@ -60,7 +70,7 @@
 
 <% if has_variable?("mapred_acls_enabled") %>
   <property>
-    <name>mapred.acls.enabled</name>
+    <name>mapreduce.cluster.acls.enabled</name>
     <value><%= mapred_acls_enabled %></value>
   </property>
 <% end %>
@@ -68,7 +78,7 @@
 <!-- specify JobTracker TaskScheduler -->
 <% if has_variable?("hadoop_jobtracker_taskscheduler") %>
   <property>
-    <name>mapred.jobtracker.taskScheduler</name>
+    <name>mapreduce.jobtracker.taskscheduler</name>
     <value><%= hadoop_jobtracker_taskscheduler %></value>
   </property>
 <% end %>
@@ -94,20 +104,46 @@
   </property>
 <% end %>
 
+<% if has_variable?("hadoop_hs_host") %>
+  <property>
+    <name>mapreduce.jobhistory.address</name>
+    <value><%= hadoop_hs_host %>:<%= hadoop_hs_port %></value>
+  </property>
+  <property>
+    <name>mapreduce.jobhistory.webapp.address</name>
+    <value><%= hadoop_hs_host %>:<%= hadoop_hs_webapp_port %></value>
+  </property>
+<% end %>
+
+  <property>
+    <name>mapreduce.framework.name</name>
+    <value>yarn</value>
+  </property>
+
   <property>
-    <name>mapred.job.tracker</name>
+    <name>mapreduce.admin.user.env</name>
+    <value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.address</name>
     <value><%= hadoop_jobtracker_host %>:<%= hadoop_jobtracker_port%></value>
   </property>
 
   <property>
-    <name>mapred.local.dir</name>
+    <name>mapreduce.cluster.local.dir</name>
     <value><%= mapred_data_dirs.join(",") %></value>
     <final>true</final>
   </property>
 
+  <!-- property>
+    <name>mapreduce.cluster.temp.dir</name>
+    <value>/mapred/system</value>
+  </property -->
+
   <property>
-    <name>mapred.child.java.opts</name>
-    <value>-Xmx1024m</value>
+    <name>mapreduce.jobtracker.system.dir</name>
+    <value>/mapred/system</value>
   </property>
 
   <property>
@@ -116,8 +152,13 @@
   </property>
 
   <property>
-    <name>mapred.system.dir</name>
-    <value>/mapred/system</value>
+    <name>yarn.app.mapreduce.am.staging-dir</name>
+    <value>/user</value>
+  </property>
+
+  <property>
+    <name>mapred.child.java.opts</name>
+    <value>-Xmx1024m</value>
   </property>
 
 <% if has_variable?("hadoop_config_mapred_child_ulimit") %>

Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml Mon Apr 16 16:10:22 2012
@@ -0,0 +1,136 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+<% if hadoop_security_authentication == "kerberos" %>
+  <!-- JobTracker security configs -->
+  <property>
+    <name>yarn.resourcemanager.principal</name>
+    <value>yarn/_HOST@<%= kerberos_realm %></value>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.keytab</name>
+    <value>/etc/yarn.keytab</value>
+  </property>
+
+  <property>
+    <name>yarn.web-proxy.principal</name>
+    <value>yarn/_HOST@<%= kerberos_realm %></value>
+  </property>
+  <property>
+    <name>yarn.web-proxy.keytab</name>
+    <value>/etc/yarn.keytab</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.principal</name>
+    <value>yarn/_HOST@<%= kerberos_realm %></value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.keytab</name>
+    <value>/etc/yarn.keytab</value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.container-executor.class</name>
+    <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.linux-container-executor.group</name>
+    <value>hadoop</value>
+  </property>
+
+  <property>
+    <name>yarn.web-proxy.principal</name>
+    <value>yarn/_HOST@<%= kerberos_realm %></value>
+  </property>
+  <property>
+    <name>yarn.web-proxy.keytab</name>
+    <value>/etc/yarn.keytab</value>
+  </property>
+<% end %>
+
+<% 
+=begin FIXME: MAPREDUCE-3916 -%> 
+  <property>
+    <name>yarn.web-proxy.address</name>
+    <value><%= hadoop_ps_host %>:<%= hadoop_ps_port %></value>
+  </property> 
+<% 
+=end -%>
+
+  <property>
+    <name>yarn.resourcemanager.resource-tracker.address</name>
+    <value><%= hadoop_rm_host %>:<%= hadoop_rt_port %></value>
+  </property>
+
+  <property>
+    <name>yarn.resourcemanager.address</name>
+    <value><%= hadoop_rm_host %>:<%= hadoop_rm_port %></value>
+  </property>
+
+  <property>
+    <name>yarn.resourcemanager.scheduler.address</name>
+    <value><%= hadoop_rm_host %>:<%= hadoop_sc_port %></value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services</name>
+    <value>mapreduce.shuffle</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+  </property>
+
+  <property>
+    <name>yarn.log-aggregation-enable</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.local-dirs</name>
+    <value><%= yarn_data_dirs.join(",") %></value>
+    <final>true</final>
+  </property>
+
+  <property>
+    <description>Where to store container logs.</description>
+    <name>yarn.nodemanager.log-dirs</name>
+    <value>/var/log/hadoop-yarn/containers</value>
+  </property>
+
+  <property>
+    <description>Where to aggregate logs to.</description>
+    <name>yarn.nodemanager.remote-app-log-dir</name>
+    <value>/var/log/hadoop-yarn/apps</value>
+  </property>
+
+  <property>
+    <description>Classpath for typical applications.</description>
+     <name>yarn.application.classpath</name>
+     <value>
+        $HADOOP_CONF_DIR,
+        $HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,
+        $HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,
+        $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,
+        $YARN_HOME/*,$YARN_HOME/lib/*
+     </value>
+  </property>
+</configuration>

Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp Mon Apr 16 16:10:22 2012
@@ -126,6 +126,11 @@ class kerberos {
   }
 
   class client inherits kerberos::site {
+    # Required for SPNEGO
+    @principal { "HTTP": 
+
+    }
+
     package { $package_name_client:
       ensure => installed,
     }
@@ -143,7 +148,7 @@ class kerberos {
     Class["kerberos::client"]
   }
 
-  define create_princs {
+  define principal {
     exec { "addprinc.$title":
        path => $kerberos::site::exec_path, # BUG: I really shouldn't need to do a FQVN here
        command => "kadmin -w secure -p kadmin/admin -q 'addprinc -randkey $title/$fqdn'",
@@ -152,23 +157,35 @@ class kerberos {
     }
   }
 
-  define host_keytab($fqdn = "$hostname.$domain", $princs = undef) {
-    $real_princs = $princs ? { 
+  define host_keytab($fqdn = "$hostname.$domain", $princs = undef, $spnego = disabled) {
+
+    require "kerberos::client"
+
+    $needed_princs = $princs ? { 
       undef   => [ $title ],
       default => $princs,
     }
  
     $keytab = "/etc/${title}.keytab"
-    $exports = inline_template("<%= real_princs.join('/$fqdn ') + '/$fqdn ' %>")
+    $exports = inline_template("<%= needed_princs.map { |x| x+'/$fqdn' }.join(' ') %>")
+    $spnego_export = $spnego ? {
+       /(true|enabled)/ => "HTTP/$fqdn",
+       default          => "",
+    }
+
+    principal { $needed_princs:
 
-    create_princs { $real_princs:
     }
 
     exec { "xst.$title":
        path => $kerberos::site::exec_path, # BUG: I really shouldn't need to do a FQVN here
-       command => "kadmin -w secure -p kadmin/admin -q 'xst -k $keytab $exports' ; chown $title $keytab",
+       command => "kadmin -w secure -p kadmin/admin -q 'xst -k $keytab $exports $spnego_export' ; chown $title $keytab",
        unless => "klist -kt $keytab 2>/dev/null | grep -q $title/$fqdn",
-       require => [ Create_princs[$real_princs] ],
+       require => [ Kerberos::Principal[$needed_princs] ],
+    }
+
+    if ($spnego =~ /(true|enabled)/) {
+      Kerberos::Principal <| title == "HTTP" |> -> Exec["xst.$title"]
     }
   }
 

Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client.pp (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client.pp?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client.pp&p1=incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client.pp Mon Apr 16 16:10:22 2012
@@ -1,12 +1,11 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-mvn clean install -DskipTests "$@" 
+class nfs::client {
+  include nfs::client::install
+  require nfs::client::service
+}

Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp&p1=incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp Mon Apr 16 16:10:22 2012
@@ -1,12 +1,11 @@
-#!/bin/sh
 # Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-mvn clean install -Dmahout.skip.distribution=false -DskipTests "$@"
-
+class nfs::client::install inherits nfs::client::params {
+  package { $package_names:
+    ensure => present,
+  }
+}

Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp Mon Apr 16 16:10:22 2012
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class nfs::client::params {
+  case "$::operatingsystem $::operatingsystemrelease" {
+    /(CentOS|RedHat) 6/: {
+      $package_names   = [ "rpcbind", "nfs-utils" ]
+      $portmap_service = "rpcbind"
+      $idmapd_service  = "rpcidmapd"
+      $nfs_version     = 4
+    }
+
+    /(CentOS|RedHat) 5/: {
+      $package_names   = [ "portmap", "nfs-utils" ]
+      $portmap_service = "portmap"
+      $nfs_version     = 3
+    }
+
+    /SLES 11/: {
+      $package_names   = [ "rpcbind", "nfs-client" ]
+      $portmap_service = "rpcbind"
+      $nfs_version     = 3
+    }
+
+    /(Debian|Ubuntu)/: {
+      $package_names   = [ "nfs-common", "portmap" ]
+      $portmap_service = "portmap"
+      $nfs_version     = 3
+    }
+
+    default: {
+      fail("nfs::client::params not supported on $::operatingsystem $::operatingsystemerelease")
+    }
+  }
+}

Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/oozie/oozie-env.sh)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp&p1=incubator/bigtop/trunk/bigtop-packages/src/common/oozie/oozie-env.sh&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/oozie/oozie-env.sh (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp Mon Apr 16 16:10:22 2012
@@ -1,13 +1,11 @@
-#! /bin/bash
-
 # Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,10 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export OOZIE_CONFIG=/etc/oozie/conf
-export OOZIE_DATA=/var/lib/oozie
-export OOZIE_LOG=/var/log/oozie
-export CATALINA_BASE=${OOZIE_DATA}/oozie-server
-export CATALINA_TMPDIR=/var/tmp/oozie
-export CATALINA_PID=/var/run/oozie/oozie.pid
+class nfs::client::service inherits nfs::client::params {
+
+  include nfs::client::install
+  
+  service { $portmap_service:
+    ensure  => running,
+    enable  => true,
+    require => Package[$package_names],
+  }
 
+  if ($nfs_version >= 4) {
+    service { $idmapd_service:
+      ensure  => running,
+      enable  => true,
+      require => [ Service[$portmap_service], Package[$package_names] ],
+      pattern => "rpc.idmapd",
+    }
+  }
+}

Propchange: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp
------------------------------------------------------------------------------
    svn:eol-style = native

Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/init.pp (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/init.pp?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/init.pp&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/init.pp Mon Apr 16 16:10:22 2012
@@ -1,11 +1,11 @@
 # Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,5 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-mapred - nofile 32768
-mapred - nproc  65536
+class nfs {
+  require nfs::client
+}

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/do-component-build (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/do-component-build?p2=incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/do-component-build&p1=incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,5 +16,10 @@
 
 set -ex
 
-mvn clean install -Dmahout.skip.distribution=false -DskipTests "$@"
+rm -f jsvc
 
+pushd unix
+  ./configure --prefix=/usr
+  make
+  cp -f jsvc ../jsvc
+popd

Added: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh Mon Apr 16 16:10:22 2012
@@ -0,0 +1,90 @@
+#!/bin/bash -x
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+usage() {
+  echo "
+usage: $0 <options>
+  Required not-so-options:
+     --build-dir=DIR             path to hive/build/dist
+     --prefix=PREFIX             path to install into
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'prefix:' \
+  -l 'build-dir:' \
+  -l 'bin-dir:' \
+  -l 'doc-dir:' \
+  -l 'man-dir:' \
+  -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+    case "$1" in
+        --prefix)
+        PREFIX=$2 ; shift 2
+        ;;
+        --build-dir)
+        BUILD_DIR=$2 ; shift 2
+        ;;
+        --bin-dir)
+        BIN_DIR=$2 ; shift 2
+        ;;
+        --man-dir)
+        MAN_DIR=$2 ; shift 2
+        ;;
+        --doc-dir)
+        DOC_DIR=$2 ; shift 2
+        ;;
+        --)
+        shift ; break
+        ;;
+        *)
+        echo "Unknown option: $1"
+        usage
+        exit 1
+        ;;
+    esac
+done
+
+for var in PREFIX BUILD_DIR; do
+  if [ -z "$(eval "echo \$$var")" ]; then
+    echo Missing param: $var
+    usage
+  fi
+done
+
+BIN_DIR=${BIN_DIR:-$PREFIX/usr/lib/bigtop-utils}
+DOC_DIR=${DOC_DIR:-$PREFIX/usr/share/doc/bigtop-jsvc}
+MAN_DIR=${MAN_DIR:-$PREFIX/usr/man}
+
+#libexec
+install -d -m 0755 ${BIN_DIR}
+cp ${BUILD_DIR}/jsvc ${BIN_DIR}
+
+# docs
+install -d -m 0755 ${DOC_DIR}
+cp ${BUILD_DIR}/*.txt  ${DOC_DIR}/
+cp ${BUILD_DIR}/README  ${DOC_DIR}/

Propchange: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh
------------------------------------------------------------------------------
    svn:executable = *

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/do-component-build (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/pig/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/do-component-build?p2=incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/do-component-build&p1=incubator/bigtop/trunk/bigtop-packages/src/common/pig/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/pig/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,6 +16,10 @@
 
 set -ex
 
-ant -Djavac.version=1.6 -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -Dversion=${PIG_BASE_VERSION} package "$@"
-
+mkdir dl
+ant -Dbase.path=dl clean download
+ant -Dbase.path=dl
+ant -Dbase.path=dl -Dskip.installer=true -f dist.xml dist-static package-tgz 
 
+mkdir build
+tar -C build --strip-components=1 -xzf output/release/v*/bin/apache-tomcat-*[0-9].tar.gz

Added: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/install_tomcat.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/install_tomcat.sh?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/install_tomcat.sh (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/install_tomcat.sh Mon Apr 16 16:10:22 2012
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+  echo "
+usage: $0 <options>
+  Required not-so-options:
+     --build-dir=DIR             path to Whirr dist.dir
+     --prefix=PREFIX             path to install into
+
+  Optional options:
+     --doc-dir=DIR               path to install docs into [/usr/share/doc/whirr]
+     --lib-dir=DIR               path to install Whirr home [/usr/lib/whirr]
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'prefix:' \
+  -l 'doc-dir:' \
+  -l 'lib-dir:' \
+  -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+    case "$1" in
+        --prefix)
+        PREFIX=$2 ; shift 2
+        ;;
+        --build-dir)
+        BUILD_DIR=$2 ; shift 2
+        ;;
+        --doc-dir)
+        DOC_DIR=$2 ; shift 2
+        ;;
+        --lib-dir)
+        LIB_DIR=$2 ; shift 2
+        ;;
+        --)
+        shift ; break
+        ;;
+        *)
+        echo "Unknown option: $1"
+        usage
+        exit 1
+        ;;
+    esac
+done
+
+for var in PREFIX BUILD_DIR ; do
+  if [ -z "$(eval "echo \$$var")" ]; then
+    echo Missing param: $var
+    usage
+  fi
+done
+
+MAN_DIR=$PREFIX/usr/share/man/man1
+DOC_DIR=${DOC_DIR:-$PREFIX/usr/share/doc/bigtop-tomcat}
+LIB_DIR=${LIB_DIR:-$PREFIX/usr/lib/bigtop-tomcat}
+
+# First we'll move everything into lib
+install -d -m 0755 $LIB_DIR
+(cd $BUILD_DIR && tar -cf - .) | (cd $LIB_DIR && tar -xf -)
+
+rm -rf $LIB_DIR/webapps
+mv -f $LIB_DIR/conf $LIB_DIR/conf.template
+
+install -d -m 0755 $DOC_DIR
+mv $LIB_DIR/{RUNNING.txt,RELEASE-NOTES,NOTICE,LICENSE} $DOC_DIR

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,4 +16,4 @@
 
 set -ex
 
-mvn clean install -DskipTests "$@" 
+mvn clean install -DskipTests -Dhadoop.profile=23 "$@" 

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/flume/flume-node.init
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/flume/flume-node.init?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/flume/flume-node.init (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/flume/flume-node.init Mon Apr 16 16:10:22 2012
@@ -15,10 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Starts a Flume node
+# Starts a Flume NG node
 #
 # chkconfig: 345 90 10
-# description: Flume node
+# description: Flume NG node
 #
 ### BEGIN INIT INFO
 # Provides:          flume-node
@@ -28,7 +28,7 @@
 # Should-Stop:
 # Default-Start:     3 4 5
 # Default-Stop:      0 1 2 6
-# Short-Description: Flume node
+# Short-Description: Flume NG node
 ### END INIT INFO
 
 . /lib/lsb/init-functions
@@ -53,7 +53,7 @@ FLUME_RUN=/var/run/flume
 FLUME_HOME=/usr/lib/flume
 
 LOCKFILE="/var/lock/subsys/flume-node"
-desc="Flume node daemon"
+desc="Flume NG node daemon"
 
 FLUME_CONF_FILE=${FLUME_CONF_FILE:-${FLUME_CONF_DIR}/flume.conf}
 EXEC_PATH=/usr/bin/flume-ng
@@ -110,19 +110,19 @@ checkstatus(){
 
   case "$status" in
     $STATUS_RUNNING)
-      log_success_msg "Flume node is running"
+      log_success_msg "Flume NG node is running"
       ;;
     $STATUS_DEAD)
-      log_failure_msg "Flume node is dead and pid file exists"
+      log_failure_msg "Flume NG node is dead and pid file exists"
       ;;
     $STATUS_DEAD_AND_LOCK)
-      log_failure_msg "Flume node is dead and lock file exists"
+      log_failure_msg "Flume NG node is dead and lock file exists"
       ;;
     $STATUS_NOT_RUNNING)
-      log_failure_msg "Flume node is not running"
+      log_failure_msg "Flume NG node is not running"
       ;;
     *)
-      log_failure_msg "Flume node status is unknown"
+      log_failure_msg "Flume NG node status is unknown"
       ;;
   esac
   return $status

Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/README
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/README?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/README (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/README Mon Apr 16 16:10:22 2012
@@ -0,0 +1,4 @@
+A minimum amount of configuration to make Hadoop daemons
+self-contained and useful. It would be really nice if the
+empty  configuration  of  Hadoop  had  the same effect by 
+default (along the lines of hbase-default.xml).

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/core-site.xml (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/core-site.xml?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/core-site.xml&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/core-site.xml Mon Apr 16 16:10:22 2012
@@ -18,8 +18,4 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
 <configuration>
-  <property>
-    <name>mapred.job.tracker</name>
-    <value>localhost:8021</value>
-  </property>
 </configuration>

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/hdfs-site.xml (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/hdfs-site.xml?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/hdfs-site.xml&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/hdfs-site.xml Mon Apr 16 16:10:22 2012
@@ -19,7 +19,7 @@
 
 <configuration>
   <property>
-    <name>mapred.job.tracker</name>
-    <value>localhost:8021</value>
+     <name>dfs.name.dir</name>
+     <value>/var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value>
   </property>
 </configuration>

Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/mapred-site.xml (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/mapred-site.xml?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/mapred-site.xml&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/mapred-site.xml Mon Apr 16 16:10:22 2012
@@ -22,4 +22,10 @@
     <name>mapred.job.tracker</name>
     <value>localhost:8021</value>
   </property>
+
+  <property>
+    <name>mapreduce.admin.user.env</name>
+    <value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
+  </property>
+
 </configuration>

Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml Mon Apr 16 16:10:22 2012
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+  <property>
+    <name>yarn.nodemanager.aux-services</name>
+    <value>mapreduce.shuffle</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+  </property>
+
+  <property>
+    <name>yarn.log-aggregation-enable</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <description>List of directories to store localized files in.</description>
+    <name>yarn.nodemanager.local-dirs</name>
+    <value>/var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir</value>
+  </property>
+
+  <property>
+    <description>Where to store container logs.</description>
+    <name>yarn.nodemanager.log-dirs</name>
+    <value>/var/log/hadoop-yarn/containers</value>
+  </property>
+
+  <property>
+    <description>Where to aggregate logs to.</description>
+    <name>yarn.nodemanager.remote-app-log-dir</name>
+    <value>/var/log/hadoop-yarn/apps</value>
+  </property>
+
+  <property>
+    <description>Classpath for typical applications.</description>
+     <name>yarn.application.classpath</name>
+     <value>
+        $HADOOP_CONF_DIR,
+        $HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,
+        $HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,
+        $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,
+        $YARN_HOME/*,$YARN_HOME/lib/*
+     </value>
+  </property>
+</configuration>

Propchange: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml
------------------------------------------------------------------------------
    svn:executable = *

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/core-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/core-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/core-site.xml Mon Apr 16 16:10:22 2012
@@ -23,11 +23,6 @@
     <value>hdfs://localhost:8020</value>
   </property>
 
-  <property>
-     <name>hadoop.tmp.dir</name>
-     <value>/var/lib/hadoop/cache/${user.name}</value>
-  </property>
-
   <!-- OOZIE proxy user setting -->
   <property>
     <name>hadoop.proxyuser.oozie.hosts</name>
@@ -38,4 +33,14 @@
     <value>*</value>
   </property>
 
+  <!-- HTTPFS proxy user setting -->
+  <property>
+    <name>hadoop.proxyuser.httpfs.hosts</name>
+    <value>*</value>
+  </property>
+  <property>
+    <name>hadoop.proxyuser.httpfs.groups</name>
+    <value>*</value>
+  </property>
+
 </configuration>

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml Mon Apr 16 16:10:22 2012
@@ -22,10 +22,6 @@
     <name>dfs.replication</name>
     <value>1</value>
   </property>
-  <property>
-     <name>dfs.permissions</name>
-     <value>false</value>
-  </property>
   <!-- Immediately exit safemode as soon as one DataNode checks in. 
        On a multi-node cluster, these configurations must be removed.  -->
   <property>
@@ -37,8 +33,19 @@
      <value>1</value>
   </property>
   <property>
-     <!-- specify this so that running 'hadoop namenode -format' formats the right dir -->
-     <name>dfs.name.dir</name>
-     <value>/var/lib/hadoop/cache/hadoop/dfs/name</value>
+     <name>hadoop.tmp.dir</name>
+     <value>/var/lib/hadoop-hdfs/cache/${user.name}</value>
+  </property>
+  <property>
+     <name>dfs.namenode.name.dir</name>
+     <value>/var/lib/hadoop-hdfs/cache/${user.name}/dfs/name</value>
+  </property>
+  <property>
+     <name>dfs.namenode.checkpoint.dir</name>
+     <value>/var/lib/hadoop-hdfs/cache/${user.name}/dfs/namesecondary</value>
+  </property>
+  <property>
+     <name>dfs.datanode.data.dir</name>
+     <value>/var/lib/hadoop-hdfs/cache/${user.name}/dfs/data</value>
   </property>
 </configuration>

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml Mon Apr 16 16:10:22 2012
@@ -22,4 +22,21 @@
     <name>mapred.job.tracker</name>
     <value>localhost:8021</value>
   </property>
+
+  <property>
+    <name>mapreduce.framework.name</name>
+    <value>yarn</value>
+  </property>
+
+  <property>
+    <name>mapreduce.admin.user.env</name>
+    <value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
+  </property>
+
+  <property>
+    <description>To set the value of tmp directory for map and reduce tasks.</description>
+    <name>mapreduce.task.tmp.dir</name>
+    <value>/var/lib/hadoop-mapreduce/cache/${user.name}/tasks</value>
+  </property>
+
 </configuration>

Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml Mon Apr 16 16:10:22 2012
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+  <property>
+    <name>yarn.nodemanager.aux-services</name>
+    <value>mapreduce.shuffle</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+  </property>
+
+  <property>
+    <name>yarn.log-aggregation-enable</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <description>List of directories to store localized files in.</description>
+    <name>yarn.nodemanager.local-dirs</name>
+    <value>/var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir</value>
+  </property>
+
+  <property>
+    <description>Where to store container logs.</description>
+    <name>yarn.nodemanager.log-dirs</name>
+    <value>/var/log/hadoop-yarn/containers</value>
+  </property>
+
+  <property>
+    <description>Where to aggregate logs to.</description>
+    <name>yarn.nodemanager.remote-app-log-dir</name>
+    <value>/var/log/hadoop-yarn/apps</value>
+  </property>
+
+  <property>
+    <description>Classpath for typical applications.</description>
+     <name>yarn.application.classpath</name>
+     <value>
+        $HADOOP_CONF_DIR,
+        $HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,
+        $HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,
+        $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,
+        $YARN_HOME/*,$YARN_HOME/lib/*
+     </value>
+  </property>
+</configuration>

Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/do-component-build
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/do-component-build?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,47 +16,28 @@
 
 set -ex
 
-# FIXME: BIGTOP-137
-chmod 777 src/c++/task-controller/configure || :
+if [ -z "$BUNDLE_SNAPPY" ] ; then
+  [ -f /usr/lib/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib"
+  [ -f /usr/lib64/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib64"
+fi
 
+mkdir build
+mkdir build/src
 
-# This is a poor man version comparison of autoconf.
-# Current autotools files target autotools 2.61 or sup.
-# There will be some build issues if we try to regenerate them with a lesser version of autotools
-# So we will only regenerate the autotools files if they are installed and their version is greater than
-# 2.60
-DO_AUTORECONF=""
-if [ -n `which autoconf` ]; then
-
-	AUTOCONF_VERSION=`autoconf --version | head -n 1 | sed -e 's/.* \([.0-9]\+\)$/\1/'`
-	echo $AUTOCONF_VERSION
-
-	if [[ $AUTOCONF_VERSION > 2.60 ]]; then
-		DO_AUTORECONF="create-c++-configure"
-	fi
-fi
+# Build artifacts
+MAVEN_SKIP_TESTS="-DskipTests -DskipTest -DskipITs"
+mvn $BUNDLE_SNAPPY -Dcontainer-executor.conf.dir=/etc/hadoop/conf -Pdist -Pnative -Pfuse -Psrc -Dtar ${MAVEN_SKIP_TESTS} install "$@"
+mvn site site:stage ${MAVEN_SKIP_TESTS} $@
+
+(cd build ; tar --strip-components=1 -xzvf  ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)
+(cd build/src ; tar --strip-components=1 -xzvf  ../../hadoop-dist/target/hadoop-${HADOOP_VERSION}-src.tar.gz)
+
+# Copy generated docs
+mkdir -p build/share/doc/
+cp -r target/site/* build/share/doc/
 
-pushd src/native
-  autoreconf -if
-popd
-
-LIBJVMS=`find ${JAVA_HOME}/jre/lib/ -name "libjvm.so" -exec dirname {} \;`
-LIBRARIES="-lcrypto -lm "
-for jvm in $LIBJVMS;
-do
-  LIBRARIES="${LIBRARIES} -L${jvm}"
-done
-
-# FIXME: the LDFLAGS="-lcrypto" is a workaround for MAPREDUCE-2127
-# FIXME: the FUSE_HOME="..." and LDFLAGS="-lm" is a workaround for HADOOP-3344
-LDFLAGS="$LIBRARIES" FUSE_HOME=`pwd`/build/c++/${HADOOP_ARCH} ant \
-  -Djava5.home=$JAVA5_HOME \
-  -Dforrest.home=$FORREST_HOME \
-  -Dversion=${HADOOP_VERSION} \
-  -Dcompile.native=true \
-  -Dhadoop.conf.dir=/etc/hadoop/conf \
-  -Dlibhdfs=1 -Dfusedfs=true -Dcompile.c++=true -Djdiff.build.dir=build/docs/jdiff \
-  $DO_AUTORECONF task-controller api-report bin-package compile-contrib package \
-  "$@"
+# Create a manifest for hadoop client package
+(cd hadoop-client/target/hadoop-client-*/share/hadoop/client/lib ; ls) > build/hadoop-client.list
 
-	
+# Copy fuse output to the build directory
+cp hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/target/fuse-dfs/src/{fuse_dfs,fuse_dfs_wrapper.sh} build/bin/



Mime
View raw message