bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1230305 - in /incubator/bigtop/branches/hadoop-0.23: bigtop-deploy/puppet/modules/hadoop/manifests/ bigtop-packages/src/common/hadoop/ bigtop-packages/src/common/hadoop/conf.pseudo/ bigtop-packages/src/deb/hadoop/ bigtop-packages/src/rpm/h...
Date Wed, 11 Jan 2012 22:51:40 GMT
Author: rvs
Date: Wed Jan 11 22:51:39 2012
New Revision: 1230305

URL: http://svn.apache.org/viewvc?rev=1230305&view=rev
Log:
BIGTOP-316. split up hadoop packages into common, hdfs, mapreduce (and yarn)

Added:
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.dirs
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst
      - copied, changed from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.preinst
      - copied, changed from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.dirs
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst
      - copied, changed from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.preinst
      - copied, changed from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.dirs
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.install
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.postinst
      - copied, changed from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.preinst
      - copied, changed from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst
Modified:
    incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.dirs
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d-yarn.tpl
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/yarn-init.tmpl
    incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Wed Jan 11 22:51:39 2012
@@ -44,6 +44,11 @@ class hadoop {
   }
 
   class common-yarn inherits common {
+    package { "hadoop-yarn":
+      ensure => latest,
+      require => [Package["jdk"], Package["hadoop"]],
+    }
+ 
     file {
       "/etc/yarn/conf/yarn-site.xml":
         content => template('hadoop/yarn-site.xml'),
@@ -52,6 +57,12 @@ class hadoop {
   }
 
   class common-hdfs inherits common {
+    package { "hadoop-hdfs":
+      ensure => latest,
+      require => [Package["jdk"], Package["hadoop"]],
+    }
+ 
+
     file {
       "/etc/hadoop/conf/core-site.xml":
         content => template('hadoop/core-site.xml'),
@@ -66,6 +77,11 @@ class hadoop {
   }
 
   class common-mapred-app inherits common-hdfs {
+    package { "hadoop-mapreduce":
+      ensure => latest,
+      require => [Package["jdk"], Package["hadoop"]],
+    }
+
     file {
       "/etc/hadoop/conf/mapred-site.xml":
         content => template('hadoop/mapred-site.xml'),
@@ -87,7 +103,7 @@ class hadoop {
 
     include common-hdfs
 
-    package { "hadoop-datanode":
+    package { "hadoop-hdfs-datanode":
       ensure => latest,
       require => Package["jdk"],
     }
@@ -99,11 +115,11 @@ class hadoop {
       #}
     }
 
-    service { "hadoop-datanode":
+    service { "hadoop-hdfs-datanode":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-datanode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [ Package["hadoop-datanode"], File[$dirs] ],
+      subscribe => [Package["hadoop-hdfs-datanode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+      require => [ Package["hadoop-hdfs-datanode"], File[$dirs] ],
     }
 
     file { $dirs:
@@ -111,7 +127,7 @@ class hadoop {
       owner => hdfs,
       group => hdfs,
       mode => 755,
-      require => [Package["hadoop"]],
+      require => [Package["hadoop-hdfs"]],
     }
   }
 
@@ -123,7 +139,7 @@ class hadoop {
       user => "hdfs",
       command => "/bin/bash -c 'hadoop fs -mkdir $title && hadoop fs -chmod $perm $title && hadoop fs -chown $user $title'",
       unless => "/bin/bash -c 'hadoop fs -ls $name >/dev/null 2>&1'",
-      require => [ Service["hadoop-namenode"], Exec["namenode format"] ],
+      require => [ Service["hadoop-hdfs-namenode"], Exec["namenode format"] ],
     }
   }
 
@@ -136,23 +152,23 @@ class hadoop {
 
     include common-hdfs
 
-    package { "hadoop-namenode":
+    package { "hadoop-hdfs-namenode":
       ensure => latest,
       require => Package["jdk"],
     }
 
-    service { "hadoop-namenode":
+    service { "hadoop-hdfs-namenode":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [Package["hadoop-namenode"], Exec["namenode format"]],
+      subscribe => [Package["hadoop-hdfs-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+      require => [Package["hadoop-hdfs-namenode"], Exec["namenode format"]],
     } 
 
     exec { "namenode format":
       user => "hdfs",
       command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
       creates => inline_template("<%= hadoop_storage_locations.split(';')[0] %>/namenode/current/VERSION"),
-      require => [ Package["hadoop-namenode"], File[$dirs] ],
+      require => [ Package["hadoop-hdfs-namenode"], File[$dirs] ],
     } 
     
     file { $dirs:
@@ -160,7 +176,7 @@ class hadoop {
       owner => hdfs,
       group => hdfs,
       mode => 700,
-      require => [Package["hadoop"]], 
+      require => [Package["hadoop-hdfs"]], 
     }
   }
 
@@ -171,16 +187,16 @@ class hadoop {
 
     include common-hdfs
 
-    package { "hadoop-secondarynamenode":
+    package { "hadoop-hdfs-secondarynamenode":
       ensure => latest,
       require => Package["jdk"],
     }
 
-    service { "hadoop-secondarynamenode":
+    service { "hadoop-hdfs-secondarynamenode":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
-      require => [Package["hadoop-secondarynamenode"]],
+      subscribe => [Package["hadoop-hdfs-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+      require => [Package["hadoop-hdfs-secondarynamenode"]],
     }
   }
 
@@ -194,16 +210,16 @@ class hadoop {
 
     include common-yarn
 
-    package { "hadoop-resourcemanager":
+    package { "hadoop-yarn-resourcemanager":
       ensure => latest,
       require => Package["jdk"],
     }
 
-    service { "hadoop-resourcemanager":
+    service { "hadoop-yarn-resourcemanager":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-resourcemanager"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/yarn/conf/yarn-site.xml"]],
-      require => [ Package["hadoop-resourcemanager"] ]
+      subscribe => [Package["hadoop-yarn-resourcemanager"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/yarn/conf/yarn-site.xml"]],
+      require => [ Package["hadoop-yarn-resourcemanager"] ]
     }
   }
 
@@ -215,16 +231,16 @@ class hadoop {
 
     include common-mapred-app
 
-    package { "hadoop-historyserver":
+    package { "hadoop-mapreduce-historyserver":
       ensure => latest,
       require => Package["jdk"],
     }
 
-    service { "hadoop-historyserver":
+    service { "hadoop-mapreduce-historyserver":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-historyserver"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/hadoop/conf/mapred-site.xml"]],
-      require => [Package["hadoop-historyserver"]],
+      subscribe => [Package["hadoop-mapreduce-historyserver"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/hadoop/conf/mapred-site.xml"]],
+      require => [Package["hadoop-mapreduce-historyserver"]],
     }
   }
 
@@ -236,16 +252,16 @@ class hadoop {
 
     include common-yarn
 
-    package { "hadoop-nodemanager":
+    package { "hadoop-yarn-nodemanager":
       ensure => latest,
       require => Package["jdk"],
     }
  
-    service { "hadoop-nodemanager":
+    service { "hadoop-yarn-nodemanager":
       ensure => running,
       hasstatus => true,
-      subscribe => [Package["hadoop-nodemanager"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/yarn/conf/yarn-site.xml"]],
-      require => [ Package["hadoop-nodemanager"], File[$dirs] ],
+      subscribe => [Package["hadoop-yarn-nodemanager"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/yarn/conf/yarn-site.xml"]],
+      require => [ Package["hadoop-yarn-nodemanager"], File[$dirs] ],
     }
 
     file { $dirs:
@@ -253,7 +269,7 @@ class hadoop {
       owner => yarn,
       group => yarn,
       mode => 755,
-      require => [Package["hadoop"]],
+      require => [Package["hadoop-yarn"]],
     }
   }
 
@@ -276,7 +292,7 @@ class hadoop {
       owner => yarn,
       group => yarn,
       mode => 755,
-      require => [Package["hadoop"]],
+      require => [Package["hadoop-mapreduce"]],
     }
   }
 
@@ -292,7 +308,7 @@ class hadoop {
       # FIXME: "hadoop-source", "hadoop-fuse", "hadoop-pipes"
       package { ["hadoop-doc", "hadoop-debuginfo", "hadoop-libhdfs"]:
         ensure => latest,
-        require => [Package["jdk"], Package["hadoop"]],  
+        require => [Package["jdk"], Package["hadoop"], Package["hadoop-hdfs"], Package["hadoop-mapreduce"]],  
       }
   }
 }

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml Wed Jan 11 22:51:39 2012
@@ -37,8 +37,19 @@
      <value>1</value>
   </property>
   <property>
-     <!-- specify this so that running 'hadoop namenode -format' formats the right dir -->
-     <name>dfs.name.dir</name>
-     <value>/var/lib/hadoop/cache/hadoop/dfs/name</value>
+     <name>hadoop.tmp.dir</name>
+     <value>/var/lib/hdfs/cache/${user.name}</value>
+  </property>
+  <property>
+     <name>dfs.namenode.name.dir</name>
+     <value>/var/lib/hdfs/cache/${user.name}/dfs/name</value>
+  </property>
+  <property>
+     <name>dfs.namenode.checkpoint.dir</name>
+     <value>/var/lib/hdfs/cache/${user.name}/dfs/namesecondary</value>
+  </property>
+  <property>
+     <name>dfs.datanode.data.dir</name>
+     <value>/var/lib/hdfs/cache/${user.name}/dfs/data</value>
   </property>
 </configuration>

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml Wed Jan 11 22:51:39 2012
@@ -33,5 +33,10 @@
     <value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
   </property>
 
+  <property>
+    <description>To set the value of tmp directory for map and reduce tasks.</description>
+    <name>mapreduce.task.tmp.dir</name>
+    <value>/var/lib/mapreduce/cache/${user.name}/tasks</value>
+  </property>
 
 </configuration>

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml Wed Jan 11 22:51:39 2012
@@ -18,7 +18,6 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 
 <configuration>
-
   <property>
     <name>yarn.nodemanager.aux-services</name>
     <value>mapreduce.shuffle</value>
@@ -29,4 +28,21 @@
     <value>org.apache.hadoop.mapred.ShuffleHandler</value>
   </property>
 
+  <property>
+    <description>List of directories to store localized files in.</description>
+    <name>yarn.nodemanager.local-dirs</name>
+    <value>/var/lib/yarn/cache/${user.name}/nm-local-dir</value>
+  </property>
+
+  <property>
+    <description>Where to store container logs.</description>
+    <name>yarn.nodemanager.log-dirs</name>
+    <value>/var/log/yarn</value>
+  </property>
+
+  <property>
+    <description>Where to aggregate logs to.</description>
+    <name>yarn.nodemanager.remote-app-log-dir</name>
+    <value>/var/log/yarn</value>
+  </property>
 </configuration>

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default Wed Jan 11 22:51:39 2012
@@ -15,7 +15,7 @@
 export HADOOP_HOME_WARN_SUPPRESS=true
 export HADOOP_HOME=/usr/lib/hadoop
 export HADOOP_PREFIX=/usr/lib/hadoop
-export HADOOP_LIBEXEC_DIR=/usr/libexec
+export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec
 export HADOOP_NAMENODE_USER=hdfs
 export HADOOP_SECONDARYNAMENODE_USER=hdfs
 export HADOOP_DATANODE_USER=hdfs
@@ -25,7 +25,7 @@ export HADOOP_IDENT_STRING=hdfs
 export HADOOP_PID_DIR=/var/run/hadoop
 export HADOOP_LOG_DIR=/var/log/hadoop
 export HADOOP_CONF_DIR=/etc/hadoop/conf
-export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HADOOP_PREFIX}/*:${HADOOP_PREFIX}/lib/*"
+export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HADOOP_PREFIX}/*:${HADOOP_PREFIX}/lib/*:${HADOOP_PREFIX}/"
 
 # Pointless definitions making yarn happy
 export HADOOP_COMMON_HOME=${HADOOP_PREFIX}

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh Wed Jan 11 22:51:39 2012
@@ -153,7 +153,7 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
 fi
 
 . /etc/default/hadoop
-. /etc/default/yarn
+[ -f /etc/default/${bin_wrapper/hadoop/yarn} ] && . /etc/default/${bin_wrapper/hadoop/yarn}
 
 # FIXME: this might need to be fixed upstream
 HADOOP_CLASSPATH="\${HADOOP_CLASSPATH}:\${YARN_CONF_DIR}"
@@ -183,11 +183,6 @@ cp ${BUILD_DIR}/share/hadoop/common/lib/
 cp ${BUILD_DIR}/share/hadoop/hdfs/lib/*.jar ${HADOOP_LIB_DIR}/
 chmod 644 ${HADOOP_LIB_DIR}/*.jar
 
-# Remove duplicate libraries:
-rm -fv ${HADOOP_LIB_DIR}/slf4j-*-1.5.11.jar
-rm -fv ${HADOOP_LIB_DIR}/stax-api-1.0.1.jar
-rm -fv ${HADOOP_LIB_DIR}/netty-3.2.3.Final.jar
-
 # hadoop jar
 install -d -m 0755 ${HADOOP_DIR}
 cp ${BUILD_DIR}/share/hadoop/mapreduce/*.jar ${HADOOP_DIR}/
@@ -239,6 +234,21 @@ for conf in conf.pseudo ; do
 done
 cp ${BUILD_DIR}/etc/hadoop/log4j.properties $HADOOP_ETC_DIR/conf.pseudo
 
+# FIXME: Provide a convenience link for configuration (HADOOP-7939)
+install -d -m 0755 ${HADOOP_DIR}/etc
+ln -s ${HADOOP_ETC_DIR##${PREFIX}}/conf ${HADOOP_DIR}/etc/hadoop
+
+# FIXME: Provide convenience links for log/run in hdfs and mapreduce (HADOOP-7939)
+install -d -m 0755 $PREFIX/var/log/ $PREFIX/var/run/
+ln -s hadoop $PREFIX/var/log/hdfs
+ln -s hadoop $PREFIX/var/run/hdfs
+ln -s hadoop $PREFIX/var/log/mapreduce
+ln -s hadoop $PREFIX/var/run/mapreduce
+
+# FIXME: The following needs to be untangled upstream (HADOOP-7939)
+cp ${BUILD_DIR}/share/hadoop/mapreduce/hadoop-mapreduce-client-shuffle*.jar ${HADOOP_DIR}/lib/
+cp ${BUILD_DIR}/share/hadoop/mapreduce/hadoop-mapreduce-client-core*.jar ${HADOOP_DIR}/lib/
+
 # Remove all hadoop test jars
 rm -fv ${HADOOP_DIR}/*test*.jar
 

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/control Wed Jan 11 22:51:39 2012
@@ -46,10 +46,44 @@ Description: A software platform for pro
  nodes around the cluster. MapReduce can then process the data where it is
  located.
 
+Package: hadoop-hdfs
+Architecture: all
+Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, bigtop-utils, hadoop (= ${source:Version})
+Description: The Hadoop Distributed File System
+ Hadoop Distributed File System (HDFS) is the primary storage system used by 
+ Hadoop applications. HDFS creates multiple replicas of data blocks and distributes 
+ them on compute nodes throughout a cluster to enable reliable, extremely rapid 
+ computations.
+
+Package: hadoop-yarn
+Architecture: all
+Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, bigtop-utils, hadoop (= ${source:Version})
+Description: The Hadoop NextGen MapReduce (YARN)
+ YARN (Hadoop NextGen MapReduce) is a general purpose data-computation framework.
+ The fundamental idea of YARN is to split up the two major functionalities of the 
+ JobTracker, resource management and job scheduling/monitoring, into separate daemons:
+ ResourceManager and NodeManager.
+ .
+ The ResourceManager is the ultimate authority that arbitrates resources among all 
+ the applications in the system. The NodeManager is a per-node slave managing allocation
+ of computational resources on a single node. Both work in support of per-application 
+ ApplicationMaster (AM).
+ .
+ An ApplicationMaster is, in effect, a framework specific library and is tasked with 
+ negotiating resources from the ResourceManager and working with the NodeManager(s) to 
+ execute and monitor the tasks. 
+
+Package: hadoop-mapreduce
+Architecture: all
+Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, bigtop-utils, hadoop-yarn (= ${source:Version})
+Description: The Hadoop MapReduce (MRv2)
+ Hadoop MapReduce is a programming model and software framework for writing applications 
+ that rapidly process vast amounts of data in parallel on large clusters of compute nodes.
+
 Package: hadoop-fuse
 Provides: hadoop-fuse
 Architecture: i386 amd64
-Depends: ${shlibs:Depends}, hadoop (= ${source:Version}), libfuse2, fuse-utils
+Depends: ${shlibs:Depends}, hadoop-hdfs (= ${source:Version}), libfuse2, fuse-utils
 Enhances: hadoop
 Description: HDFS exposed over a Filesystem in Userspace
  These projects (enumerated below) allow HDFS to be mounted (on most flavors 
@@ -75,49 +109,57 @@ Description: Source code for Hadoop
 Package: hadoop-conf-pseudo
 Provides: hadoop-conf-pseudo
 Architecture: all
-Depends: hadoop (= ${source:Version}), hadoop-namenode (= ${source:Version}), hadoop-datanode (= ${source:Version}), hadoop-secondarynamenode (= ${source:Version}), hadoop-resourcemanager (= ${source:Version}), hadoop-nodemanager (= ${source:Version})
+Depends: hadoop (= ${source:Version}), hadoop-hdfs-namenode (= ${source:Version}), hadoop-hdfs-datanode (= ${source:Version}), hadoop-hdfs-secondarynamenode (= ${source:Version}), hadoop-yarn-resourcemanager (= ${source:Version}), hadoop-yarn-nodemanager (= ${source:Version}), hadoop-mapreduce-historyserver (= ${source:Version})
 Description: Pseudo-distributed Hadoop configuration
  Contains configuration files for a "pseudo-distributed" Hadoop deployment.
  In this mode, each of the hadoop components runs as a separate Java process,
  but all on the same machine.
 
-Package: hadoop-nodemanager
-Provides: hadoop-nodemanager
+Package: hadoop-mapreduce-historyserver
+Provides: hadoop-mapreduce-historyserver
 Architecture: all
-Depends: hadoop (= ${source:Version})
+Depends: hadoop-mapreduce (= ${source:Version})
+Description: MapReduce History Server
+ The History server keeps records of the different activities being performed 
+ on a Apache Hadoop cluster.
+
+Package: hadoop-yarn-nodemanager
+Provides: hadoop-yarn-nodemanager
+Architecture: all
+Depends: hadoop-yarn (= ${source:Version})
 Description: Node manager for Hadoop
  The NodeManager is the per-machine framework agent who is responsible for
  containers, monitoring their resource usage (cpu, memory, disk, network) and
  reporting the same to the ResourceManager/Scheduler.
 
-Package: hadoop-resourcemanager
-Provides: hadoop-resourcemanager
+Package: hadoop-yarn-resourcemanager
+Provides: hadoop-yarn-resourcemanager
 Architecture: all
-Depends: hadoop (= ${source:Version})
+Depends: hadoop-yarn (= ${source:Version})
 Description: Resource manager for Hadoop
  The resource manager manages the global assignment of compute resources to applications.
 
-Package: hadoop-namenode
-Provides: hadoop-namenode
+Package: hadoop-hdfs-namenode
+Provides: hadoop-hdfs-namenode
 Architecture: all
-Depends: hadoop (= ${source:Version})
+Depends: hadoop-hdfs (= ${source:Version})
 Description: Name Node for Hadoop
  The Hadoop Distributed Filesystem (HDFS) requires one unique server, the
  namenode, which manages the block locations of files on the filesystem.
 
-Package: hadoop-secondarynamenode
-Provides: hadoop-secondarynamenode
+Package: hadoop-hdfs-secondarynamenode
+Provides: hadoop-hdfs-secondarynamenode
 Architecture: all
-Depends: hadoop (= ${source:Version})
+Depends: hadoop-hdfs (= ${source:Version})
 Description: Secondary Name Node for Hadoop
  The Secondary Name Node is responsible for checkpointing file system images.
  It is _not_ a failover pair for the namenode, and may safely be run on the
  same machine.
 
-Package: hadoop-datanode
-Provides: hadoop-datanode
+Package: hadoop-hdfs-datanode
+Provides: hadoop-hdfs-datanode
 Architecture: all
-Depends: hadoop (= ${source:Version})
+Depends: hadoop-hdfs (= ${source:Version})
 Description: Data Node for Hadoop
  The Data Nodes in the Hadoop Cluster are responsible for serving up
  blocks of data over the network to Hadoop Distributed Filesystem

Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.dirs
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.dirs?rev=1230305&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.dirs (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.dirs Wed Jan 11 22:51:39 2012
@@ -0,0 +1,6 @@
+/etc/conf.empty/
+/usr/lib/hadoop
+/usr/bin
+/var/lib/hdfs/cache
+# /var/log/hdfs
+# /var/run/hdfs

Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install?rev=1230305&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.install Wed Jan 11 22:51:39 2012
@@ -0,0 +1,21 @@
+/etc/hadoop/conf.empty/hdfs-site.xml
+/etc/hadoop/conf.empty/httpfs-*
+/usr/lib/hadoop/hadoop-hdfs*.jar
+/usr/lib/hadoop/libexec/hdfs-config.sh
+/usr/lib/hadoop/libexec/httpfs-config.sh
+/usr/lib/hadoop/webapps
+/usr/lib/hadoop/sbin/update-hdfs-env.sh
+/usr/lib/hadoop/sbin/start-secure-dns.sh
+/usr/lib/hadoop/sbin/stop-secure-dns.sh
+/usr/lib/hadoop/sbin/start-balancer.sh
+/usr/lib/hadoop/sbin/stop-balancer.sh
+/usr/lib/hadoop/sbin/start-dfs.sh
+/usr/lib/hadoop/sbin/stop-dfs.sh
+/usr/lib/hadoop/sbin/refresh-namenodes.sh
+/usr/lib/hadoop/sbin/distribute-exclude.sh
+/usr/lib/hadoop/sbin/httpfs.sh
+/usr/lib/hadoop/bin/hdfs
+/usr/bin/hdfs
+# FIXME: HADOOP-7939
+/var/log/hdfs
+/var/run/hdfs

Copied: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst (from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst?p2=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst&p1=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default&r1=1230210&r2=1230305&rev=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.postinst Wed Jan 11 22:51:39 2012
@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,21 +14,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_PREFIX=/usr/lib/hadoop
-export HADOOP_LIBEXEC_DIR=/usr/libexec
-export HADOOP_NAMENODE_USER=hdfs
-export HADOOP_SECONDARYNAMENODE_USER=hdfs
-export HADOOP_DATANODE_USER=hdfs
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hdfs
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
-export HADOOP_CONF_DIR=/etc/hadoop/conf
-export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HADOOP_PREFIX}/*:${HADOOP_PREFIX}/lib/*"
-
-# Pointless definitions making yarn happy
-export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
-export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
+
+# postinst script for hadoop
+
+set -e
+
+case "$1" in
+    configure)
+        # FIXME: HADOOP-7939
+	# mkdir -p /var/log/hdfs /var/run/hdfs || :
+	# chgrp -R hadoop /var/log/hdfs /var/run/hdfs
+	# chmod g+w /var/run/hdfs /var/log/hdfs
+        chown hdfs:hadoop /var/lib/hdfs/ /var/lib/hdfs/cache
+	chmod 1777 /var/lib/hdfs/cache
+    ;;
+
+    abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+#DEBHELPER#

Copied: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.preinst (from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.preinst?p2=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.preinst&p1=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst&r1=1230210&r2=1230305&rev=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-hdfs.preinst Wed Jan 11 22:51:39 2012
@@ -32,28 +32,17 @@ set -e
 
 case "$1" in
     install|upgrade)
-        addgroup --system hadoop >/dev/null 2>/dev/null || :
         if ! getent passwd hdfs >/dev/null; then
             # Adding system user: hdfs.
             adduser \
                 --system \
                 --group \
-                --home /usr/lib/hadoop \
+                --home /var/lib/hdfs \
                 --gecos "Hadoop HDFS" \
                 --shell /bin/bash \
                 hdfs >/dev/null 2>/dev/null || :
             usermod --append --groups hadoop hdfs || :
         fi
-        if ! getent passwd mapred >/dev/null; then
-            adduser \
-                --system \
-                --group \
-                --home /usr/lib/hadoop \
-                --gecos "Hadoop MapReduce" \
-                --shell /bin/bash \
-                mapred >/dev/null 2>/dev/null || :
-            usermod --append --groups hadoop mapred || :
-        fi
     ;;
 
     abort-upgrade)

Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.dirs
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.dirs?rev=1230305&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.dirs (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.dirs Wed Jan 11 22:51:39 2012
@@ -0,0 +1,6 @@
+/etc/conf.empty/
+/usr/lib/hadoop
+/usr/bin
+/var/lib/mapreduce/cache
+# /var/log/mapreduce
+# /var/run/mapreduce

Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install?rev=1230305&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.install Wed Jan 11 22:51:39 2012
@@ -0,0 +1,7 @@
+/usr/lib/hadoop/hadoop-mapreduce*.jar
+/usr/lib/hadoop/libexec/mapred-config.sh
+/usr/lib/hadoop/bin/mapred
+/usr/bin/mapred
+# FIXME: HADOOP-7939
+/var/log/mapreduce
+/var/run/mapreduce

Copied: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst (from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst?p2=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst&p1=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default&r1=1230210&r2=1230305&rev=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.postinst Wed Jan 11 22:51:39 2012
@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,21 +14,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_PREFIX=/usr/lib/hadoop
-export HADOOP_LIBEXEC_DIR=/usr/libexec
-export HADOOP_NAMENODE_USER=hdfs
-export HADOOP_SECONDARYNAMENODE_USER=hdfs
-export HADOOP_DATANODE_USER=hdfs
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hdfs
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
-export HADOOP_CONF_DIR=/etc/hadoop/conf
-export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HADOOP_PREFIX}/*:${HADOOP_PREFIX}/lib/*"
-
-# Pointless definitions making yarn happy
-export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
-export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
+
+# postinst script for hadoop
+
+set -e
+
+case "$1" in
+    configure)
+        # FIXME: HADOOP-7939
+	# mkdir -p /var/log/mapreduce /var/run/mapreduce || :
+	# chgrp -R hadoop /var/log/mapreduce /var/run/mapreduce
+	# chmod g+w /var/run/mapreduce /var/log/mapreduce
+        chown mapreduce:hadoop /var/lib/mapreduce /var/lib/mapreduce/cache
+	chmod 1777 /var/lib/mapreduce/cache
+    ;;
+
+    abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+#DEBHELPER#

Copied: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.preinst (from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.preinst?p2=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.preinst&p1=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst&r1=1230210&r2=1230305&rev=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-mapreduce.preinst Wed Jan 11 22:51:39 2012
@@ -32,27 +32,15 @@ set -e
 
 case "$1" in
     install|upgrade)
-        addgroup --system hadoop >/dev/null 2>/dev/null || :
-        if ! getent passwd hdfs >/dev/null; then
-            # Adding system user: hdfs.
+        if ! getent passwd mapreduce >/dev/null; then
             adduser \
                 --system \
                 --group \
-                --home /usr/lib/hadoop \
-                --gecos "Hadoop HDFS" \
-                --shell /bin/bash \
-                hdfs >/dev/null 2>/dev/null || :
-            usermod --append --groups hadoop hdfs || :
-        fi
-        if ! getent passwd mapred >/dev/null; then
-            adduser \
-                --system \
-                --group \
-                --home /usr/lib/hadoop \
+                --home /var/lib/mapreduce \
                 --gecos "Hadoop MapReduce" \
                 --shell /bin/bash \
-                mapred >/dev/null 2>/dev/null || :
-            usermod --append --groups hadoop mapred || :
+                mapreduce >/dev/null 2>/dev/null || :
+            usermod --append --groups hadoop mapreduce || :
         fi
     ;;
 

Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.dirs
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.dirs?rev=1230305&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.dirs (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.dirs Wed Jan 11 22:51:39 2012
@@ -0,0 +1,6 @@
+/etc/conf.empty/
+/usr/lib/hadoop
+/usr/bin
+/var/lib/yarn/cache
+/var/log/yarn
+/var/run/yarn

Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.install?rev=1230305&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.install (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.install Wed Jan 11 22:51:39 2012
@@ -0,0 +1,13 @@
+/etc/hadoop/conf.empty/yarn-env.sh
+/etc/hadoop/conf.empty/yarn-site.xml
+/etc/hadoop/conf.empty/mrapp-generated-classpath
+/etc/default/yarn
+/usr/lib/hadoop/hadoop-yarn*.jar
+/usr/lib/hadoop/libexec/yarn-config.sh
+/usr/lib/hadoop/sbin/start-yarn.sh
+/usr/lib/hadoop/sbin/stop-yarn.sh
+/usr/lib/hadoop/sbin/yarn-daemon.sh
+/usr/lib/hadoop/sbin/yarn-daemons.sh
+/usr/lib/hadoop/bin/yarn
+/usr/lib/hadoop/bin/container-executor
+/usr/bin/yarn

Copied: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.postinst (from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.postinst?p2=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.postinst&p1=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default&r1=1230210&r2=1230305&rev=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/hadoop.default (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.postinst Wed Jan 11 22:51:39 2012
@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,21 +14,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_PREFIX=/usr/lib/hadoop
-export HADOOP_LIBEXEC_DIR=/usr/libexec
-export HADOOP_NAMENODE_USER=hdfs
-export HADOOP_SECONDARYNAMENODE_USER=hdfs
-export HADOOP_DATANODE_USER=hdfs
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hdfs
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
-export HADOOP_CONF_DIR=/etc/hadoop/conf
-export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HADOOP_PREFIX}/*:${HADOOP_PREFIX}/lib/*"
-
-# Pointless definitions making yarn happy
-export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
-export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
+
+# postinst script for hadoop
+
+set -e
+
+case "$1" in
+    configure)
+	mkdir -p /var/log/yarn /var/run/yarn /var/lib/yarn/cache || :
+	chgrp -R hadoop /var/log/yarn /var/run/yarn
+	chmod g+w /var/log/yarn /var/run/yarn
+        chown yarn:hadoop /var/lib/yarn/ /var/lib/yarn/cache
+	chmod 1777 /var/lib/yarn/cache
+    ;;
+
+    abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+#DEBHELPER#

Copied: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.preinst (from r1230210, incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst)
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.preinst?p2=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.preinst&p1=incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst&r1=1230210&r2=1230305&rev=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop-yarn.preinst Wed Jan 11 22:51:39 2012
@@ -32,27 +32,16 @@ set -e
 
 case "$1" in
     install|upgrade)
-        addgroup --system hadoop >/dev/null 2>/dev/null || :
-        if ! getent passwd hdfs >/dev/null; then
-            # Adding system user: hdfs.
+        if ! getent passwd yarn >/dev/null; then
+            # Adding system user: yarn.
             adduser \
                 --system \
                 --group \
-                --home /usr/lib/hadoop \
-                --gecos "Hadoop HDFS" \
+                --home /var/lib/yarn \
+                --gecos "Hadoop YARN" \
                 --shell /bin/bash \
-                hdfs >/dev/null 2>/dev/null || :
-            usermod --append --groups hadoop hdfs || :
-        fi
-        if ! getent passwd mapred >/dev/null; then
-            adduser \
-                --system \
-                --group \
-                --home /usr/lib/hadoop \
-                --gecos "Hadoop MapReduce" \
-                --shell /bin/bash \
-                mapred >/dev/null 2>/dev/null || :
-            usermod --append --groups hadoop mapred || :
+                yarn >/dev/null 2>/dev/null || :
+            usermod --append --groups hadoop yarn || :
         fi
     ;;
 

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.dirs
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.dirs?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.dirs (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.dirs Wed Jan 11 22:51:39 2012
@@ -1,8 +1,7 @@
+/etc/conf.empty/
 /usr/lib/hadoop
-/var/log/hadoop
 /usr/bin
-/var/run/hadoop
-/var/lib/hadoop
-/var/lib/hadoop/cache/hadoop
 /usr/share/doc/hadoop
-/usr/libexec 
\ No newline at end of file
+/var/lib/hadoop/cache/
+/var/log/hadoop
+/var/run/hadoop

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.install Wed Jan 11 22:51:39 2012
@@ -1,9 +1,23 @@
-/usr/bin
-/usr/libexec
-/usr/lib/hadoop
-/etc/hadoop/conf.empty
+/etc/hadoop/conf.empty/hadoop-metrics.properties
+/etc/hadoop/conf.empty/hadoop-metrics2.properties
+/etc/hadoop/conf.empty/log4j.properties
+/etc/hadoop/conf.empty/slaves
+/etc/hadoop/conf.empty/ssl-client.xml.example
+/etc/hadoop/conf.empty/ssl-server.xml.example
 /etc/default/hadoop
-/etc/default/yarn
 /etc/security/limits.d/hadoop.nofiles.conf
-#/usr/share/doc/hadoop
-/usr/lib/hadoop/lib/native
+/usr/lib/hadoop/hadoop-common*.jar
+/usr/lib/hadoop/hadoop-auth*.jar
+/usr/lib/hadoop/hadoop-annotations*.jar
+/usr/lib/hadoop/lib
+/usr/lib/hadoop/etc
+/usr/lib/hadoop/libexec/hadoop-config.sh
+/usr/lib/hadoop/sbin/hadoop-*.sh
+/usr/lib/hadoop/sbin/update-hadoop-env.sh
+/usr/lib/hadoop/sbin/slaves.sh
+/usr/lib/hadoop/sbin/start-all.sh
+/usr/lib/hadoop/sbin/stop-all.sh
+/usr/lib/hadoop/bin/hadoop
+/usr/lib/hadoop/bin/rcc
+/usr/bin/hadoop
+/usr/share/man/man1/hadoop.1.*

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/hadoop.preinst Wed Jan 11 22:51:39 2012
@@ -33,27 +33,6 @@ set -e
 case "$1" in
     install|upgrade)
         addgroup --system hadoop >/dev/null 2>/dev/null || :
-        if ! getent passwd hdfs >/dev/null; then
-            # Adding system user: hdfs.
-            adduser \
-                --system \
-                --group \
-                --home /usr/lib/hadoop \
-                --gecos "Hadoop HDFS" \
-                --shell /bin/bash \
-                hdfs >/dev/null 2>/dev/null || :
-            usermod --append --groups hadoop hdfs || :
-        fi
-        if ! getent passwd mapred >/dev/null; then
-            adduser \
-                --system \
-                --group \
-                --home /usr/lib/hadoop \
-                --gecos "Hadoop MapReduce" \
-                --shell /bin/bash \
-                mapred >/dev/null 2>/dev/null || :
-            usermod --append --groups hadoop mapred || :
-        fi
     ;;
 
     abort-upgrade)

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/rules Wed Jan 11 22:51:39 2012
@@ -60,9 +60,10 @@ install: build
 	  --distro-dir=debian \
 	  --build-dir=${PWD}/build \
 	  --system-lib-dir=debian/tmp/usr/lib/ \
-	  --system-libexec-dir=debian/tmp/usr/libexec/ \
+	  --system-libexec-dir=debian/tmp/usr/lib/hadoop/libexec/ \
 	  --system-include-dir=debian/tmp/usr/include \
 	  --doc-dir=debian/tmp/usr/share/doc/hadoop-doc \
+          --man-dir=debian/tmp/usr/share/man \
 	  --example-dir=debian/tmp/usr/share/doc/hadoop/examples \
 	  --native-build-string=${native_dir} \
 	  --installed-lib-dir=/usr/lib/hadoop
@@ -77,22 +78,22 @@ datanode_user=hdfs
 resourcemanager_user=yarn
 nodemanager_user=yarn
 
-namenode secondarynamenode datanode: debian/service-init.d.tpl
-	sed -e "s|@HADOOP_DAEMON@|$@|" \
+hdfs-namenode hdfs-secondarynamenode hdfs-datanode mapreduce-historyserver: debian/service-init.d.tpl
+	sed -e "s|@HADOOP_DAEMON@|$(patsubst mapreduce-%,%,$(patsubst hdfs-%,%,$@))|" \
             -e "s|@HADOOP_MAJOR_VERSION@|$hadoop_version|" \
 	    -e "s|@DAEMON_USER@|$($@_user)|" $< > debian/hadoop-$@.init
 	# FIXME: workaround for BIGTOP-105
 	sed -e "s|@HADOOP_DAEMON@|$@|" < debian/hadoop.daemon.postinst.tpl > debian/hadoop-$@.postinst
 
-resourcemanager nodemanager: debian/service-init.d-yarn.tpl
-	sed -e "s|@HADOOP_DAEMON@|$@|" \
+yarn-resourcemanager yarn-nodemanager: debian/service-init.d-yarn.tpl
+	sed -e "s|@HADOOP_DAEMON@|$(patsubst yarn-%,%,$@)|" \
             -e "s|@HADOOP_MAJOR_VERSION@|$hadoop_version|" \
 	    -e "s|@DAEMON_USER@|$($@_user)|" $< > debian/hadoop-$@.init
 	# FIXME: workaround for BIGTOP-105
 	sed -e "s|@HADOOP_DAEMON@|$@|" < debian/hadoop.daemon.postinst.tpl > debian/hadoop-$@.postinst
 
 
-install-indep: namenode secondarynamenode datanode resourcemanager nodemanager
+install-indep: hdfs-namenode hdfs-secondarynamenode hdfs-datanode yarn-resourcemanager yarn-nodemanager mapreduce-historyserver
 	mkdir -p debian/tmp/etc/default
 	cp debian/hadoop.default debian/tmp/etc/default/hadoop
 	cp debian/yarn.default debian/tmp/etc/default/yarn

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d-yarn.tpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d-yarn.tpl?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d-yarn.tpl (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d-yarn.tpl Wed Jan 11 22:51:39 2012
@@ -56,16 +56,8 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
   . /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
-. /usr/libexec/hadoop-config.sh
-. /usr/libexec/yarn-config.sh 
-
-# FIXME: this needs to be removed once hadoop-config.sh stop clobbering HADOOP_HOME
-. /etc/default/hadoop
-. /etc/default/yarn
-
-
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-DAEMON_SCRIPT=$HADOOP_HOME/bin/yarn-daemon.sh
+DAEMON_SCRIPT=$HADOOP_HOME/sbin/yarn-daemon.sh
 NAME=hadoop-@HADOOP_DAEMON@
 DESC="Hadoop @HADOOP_DAEMON@ daemon"
 PID_FILE=$YARN_PID_DIR/hadoop-$YARN_IDENT_STRING-@HADOOP_DAEMON@.pid
@@ -155,13 +147,13 @@ hadoop_stop_pidfile() {
 start() {
     TARGET_USER_NAME="YARN_`echo @HADOOP_DAEMON@ | tr a-z A-Z`_USER"
     TARGET_USER=$(eval "echo \$$TARGET_USER_NAME")
-    su -s /bin/bash $TARGET_USER -c "$HADOOP_HOME/bin/yarn-daemon.sh start @HADOOP_DAEMON@ $DAEMON_FLAGS"
+    su -s /bin/bash $TARGET_USER -c "$DAEMON_SCRIPT start @HADOOP_DAEMON@ $DAEMON_FLAGS"
 
     # Some processes are slow to start
     sleep $SLEEP_TIME
 }
 stop() {
-    $HADOOP_HOME/bin/yarn-daemon.sh stop @HADOOP_DAEMON@
+    $DAEMON_SCRIPT stop @HADOOP_DAEMON@
 }
 
 check_for_root() {

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hadoop/service-init.d.tpl Wed Jan 11 22:51:39 2012
@@ -47,7 +47,6 @@
 install -d -m 0775 -o root -g hadoop /var/run/hadoop
 
 . /etc/default/hadoop
-. /etc/default/yarn
 
 # Autodetect JAVA_HOME if not defined
 if [ -e /usr/libexec/bigtop-detect-javahome ]; then
@@ -56,14 +55,6 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
   . /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
-. /usr/libexec/hadoop-config.sh
-. /usr/libexec/yarn-config.sh 
-
-# FIXME: this needs to be removed once hadoop-config.sh stop clobbering HADOOP_HOME
-. /etc/default/hadoop
-. /etc/default/yarn
-
-
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 DAEMON_SCRIPT=$HADOOP_HOME/sbin/hadoop-daemon.sh
 NAME=hadoop-@HADOOP_DAEMON@

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl Wed Jan 11 22:51:39 2012
@@ -30,10 +30,6 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
   . /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
-. /usr/libexec/hadoop-config.sh
-# FIXME: this needs to be removed once hadoop-config.sh stop clobbering HADOOP_HOME
-. /etc/default/hadoop
-
 RETVAL_SUCCESS=0
 
 STATUS_RUNNING=0

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/hadoop-init.tmpl.suse Wed Jan 11 22:51:39 2012
@@ -41,11 +41,6 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
   . /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
-. /usr/libexec/hadoop-config.sh 
-
-# FIXME: this needs to be removed once hadoop-config.sh stop clobbering HADOOP_HOME
-. /etc/default/hadoop
-
 RETVAL_SUCCESS=0
 
 STATUS_RUNNING=0

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/yarn-init.tmpl
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/yarn-init.tmpl?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/yarn-init.tmpl (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SOURCES/yarn-init.tmpl Wed Jan 11 22:51:39 2012
@@ -42,12 +42,6 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-d
   . /usr/lib/bigtop-utils/bigtop-detect-javahome
 fi
 
-. /usr/libexec/yarn-config.sh 
-
-# FIXME: this needs to be removed once yarn-config.sh stop clobbering YARN_HOME
-. /etc/default/hadoop
-. /etc/default/yarn
-
 RETVAL_SUCCESS=0
 
 STATUS_RUNNING=0

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec?rev=1230305&r1=1230304&r2=1230305&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec Wed Jan 11 22:51:39 2012
@@ -31,11 +31,25 @@
 %define log_hadoop_dirname /var/log
 %define log_hadoop %{log_hadoop_dirname}/%{name}
 %define log_yarn %{log_hadoop_dirname}/yarn
+%define log_hdfs %{log_hadoop_dirname}/hdfs
+%define log_mapreduce %{log_hadoop_dirname}/mapreduce
+%define run_hadoop_dirname /var/run
+%define run_hadoop %{run_hadoop_dirname}/hadoop
+%define run_yarn %{run_hadoop_dirname}/yarn
+%define run_hdfs %{run_hadoop_dirname}/hdfs
+%define run_mapreduce %{run_hadoop_dirname}/mapreduce
+%define state_hadoop_dirname /var/lib
+%define state_hadoop %{state_hadoop_dirname}/hadoop
+%define state_yarn %{state_hadoop_dirname}/yarn
+%define state_hdfs %{state_hadoop_dirname}/hdfs
+%define state_mapreduce %{state_hadoop_dirname}/mapreduce
 %define bin_hadoop %{_bindir}
 %define man_hadoop %{_mandir}
 %define doc_hadoop %{_docdir}/%{name}-%{hadoop_version}
-%define hadoop_services namenode secondarynamenode datanode
-%define yarn_services resourcemanager nodemanager historyserver
+%define mapreduce_services mapreduce-historyserver
+%define hdfs_services hdfs-namenode hdfs-secondarynamenode hdfs-datanode
+%define yarn_services yarn-resourcemanager yarn-nodemanager
+%define hadoop_services %{hdfs_services} %{mapreduce_services} %{yarn_services}
 # Hadoop outputs built binaries into %{hadoop_build}
 %define hadoop_build_path build
 %define static_images_dir src/webapps/static/images
@@ -172,68 +186,109 @@ multiple replicas of data blocks for rel
 nodes around the cluster. MapReduce can then process the data where it is
 located.
 
+%package hdfs
+Summary: The Hadoop Distributed File System
+Group: System/Daemons
+Requires: %{name} = %{version}-%{release}
 
-%package namenode
-Summary: The Hadoop namenode manages the block locations of HDFS files
+%description hdfs
+Hadoop Distributed File System (HDFS) is the primary storage system used by 
+Hadoop applications. HDFS creates multiple replicas of data blocks and distributes 
+them on compute nodes throughout a cluster to enable reliable, extremely rapid 
+computations.
+
+%package yarn
+Summary: The Hadoop NextGen MapReduce (YARN)
 Group: System/Daemons
 Requires: %{name} = %{version}-%{release}
 
-%description namenode
+%description yarn
+YARN (Hadoop NextGen MapReduce) is a general purpose data-computation framework.
+The fundamental idea of YARN is to split up the two major functionalities of the 
+JobTracker, resource management and job scheduling/monitoring, into separate daemons:
+ResourceManager and NodeManager.
+
+The ResourceManager is the ultimate authority that arbitrates resources among all 
+the applications in the system. The NodeManager is a per-node slave managing allocation
+of computational resources on a single node. Both work in support of per-application 
+ApplicationMaster (AM).
+
+An ApplicationMaster is, in effect, a framework specific library and is tasked with 
+negotiating resources from the ResourceManager and working with the NodeManager(s) to 
+execute and monitor the tasks. 
+
+
+%package mapreduce
+Summary: The Hadoop MapReduce (MRv2)
+Group: System/Daemons
+Requires: %{name}-yarn = %{version}-%{release}
+
+%description mapreduce
+Hadoop MapReduce is a programming model and software framework for writing applications 
+that rapidly process vast amounts of data in parallel on large clusters of compute nodes.
+
+
+%package hdfs-namenode
+Summary: The Hadoop namenode manages the block locations of HDFS files
+Group: System/Daemons
+Requires: %{name}-hdfs = %{version}-%{release}
+
+%description hdfs-namenode
 The Hadoop Distributed Filesystem (HDFS) requires one unique server, the
 namenode, which manages the block locations of files on the filesystem.
 
 
-%package secondarynamenode
+%package hdfs-secondarynamenode
 Summary: Hadoop Secondary namenode
 Group: System/Daemons
-Requires: %{name} = %{version}-%{release}
+Requires: %{name}-hdfs = %{version}-%{release}
 
-%description secondarynamenode
+%description hdfs-secondarynamenode
 The Secondary Name Node periodically compacts the Name Node EditLog
 into a checkpoint.  This compaction ensures that Name Node restarts
 do not incur unnecessary downtime.
 
 
-%package datanode
+%package hdfs-datanode
 Summary: Hadoop Data Node
 Group: System/Daemons
-Requires: %{name} = %{version}-%{release}
+Requires: %{name}-hdfs = %{version}-%{release}
 
-%description datanode
+%description hdfs-datanode
 The Data Nodes in the Hadoop Cluster are responsible for serving up
 blocks of data over the network to Hadoop Distributed Filesystem
 (HDFS) clients.
 
-%package resourcemanager
+%package yarn-resourcemanager
 Summary: Yarn Resource Manager
 Group: System/Daemons
-Requires: %{name} = %{version}-%{release}
+Requires: %{name}-yarn = %{version}-%{release}
 
-%description resourcemanager
+%description yarn-resourcemanager
 The resource manager manages the global assignment of compute resources to applications
 
-%package nodemanager
+%package yarn-nodemanager
 Summary: Yarn Node Manager
 Group: System/Daemons
-Requires: %{name} = %{version}-%{release}
+Requires: %{name}-yarn = %{version}-%{release}
 
-%description nodemanager
+%description yarn-nodemanager
 The NodeManager is the per-machine framework agent who is responsible for
 containers, monitoring their resource usage (cpu, memory, disk, network) and
 reporting the same to the ResourceManager/Scheduler.
 
-%package historyserver
-Summary: Yarn History Server
+%package mapreduce-historyserver
+Summary: MapReduce History Server
 Group: System/Daemons
-Requires: %{name} = %{version}-%{release}
+Requires: %{name}-mapreduce = %{version}-%{release}
 
-%description historyserver
+%description mapreduce-historyserver
 The History server keeps records of the different activities being performed on a Apache Hadoop cluster
 
 %package conf-pseudo
 Summary: Hadoop installation in pseudo-distributed mode
 Group: System/Daemons
-Requires: %{name} = %{version}-%{release}, %{name}-namenode = %{version}-%{release}, %{name}-datanode = %{version}-%{release}, %{name}-secondarynamenode = %{version}-%{release}, %{name}-resourcemanager = %{version}-%{release}, %{name}-nodemanager = %{version}-%{release}, %{name}-historyserver = %{version}-%{release}
+Requires: %{name} = %{version}-%{release}, %{name}-hdfs-namenode = %{version}-%{release}, %{name}-hdfs-datanode = %{version}-%{release}, %{name}-hdfs-secondarynamenode = %{version}-%{release}, %{name}-yarn-resourcemanager = %{version}-%{release}, %{name}-yarn-nodemanager = %{version}-%{release}, %{name}-mapreduce-historyserver = %{version}-%{release}
 
 %description conf-pseudo
 Installation of this RPM will setup your machine to run in pseudo-distributed mode
@@ -249,7 +304,7 @@ Documentation for Hadoop
 %package libhdfs
 Summary: Hadoop Filesystem Library
 Group: Development/Libraries
-Requires: %{name} = %{version}-%{release}
+Requires: %{name}-hdfs = %{version}-%{release}
 # TODO: reconcile libjvm
 AutoReq: no
 
@@ -283,7 +338,7 @@ bash %{SOURCE2} \
   --build-dir=$PWD/build \
   --system-include-dir=$RPM_BUILD_ROOT%{_includedir} \
   --system-lib-dir=$RPM_BUILD_ROOT%{_libdir} \
-  --system-libexec-dir=$RPM_BUILD_ROOT%{libexecdir} \
+  --system-libexec-dir=$RPM_BUILD_ROOT/%{lib_hadoop}/libexec \
   --hadoop-etc-dir=$RPM_BUILD_ROOT%{etc_hadoop} \
   --prefix=$RPM_BUILD_ROOT \
   --doc-dir=$RPM_BUILD_ROOT%{doc_hadoop} \
@@ -305,12 +360,12 @@ orig_init_file=$RPM_SOURCE_DIR/hadoop-in
 yarn_orig_init_file=$RPM_SOURCE_DIR/yarn-init.tmpl
 
 # Generate the init.d scripts
-for service in %{hadoop_services}
+for service in %{hdfs_services} %{mapreduce_services}
 do
        init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{name}-${service}
        %__cp $orig_init_file $init_file
        %__sed -i -e 's|@HADOOP_COMMON_ROOT@|%{lib_hadoop}|' $init_file
-       %__sed -i -e "s|@HADOOP_DAEMON@|${service}|" $init_file
+       %__sed -i -e "s|@HADOOP_DAEMON@|${service#*-}|" $init_file
        %__sed -i -e 's|@HADOOP_CONF_DIR@|%{config_hadoop}|' $init_file
        %__sed -i -e 's|@HADOOP_DAEMON_USER@|hdfs|' $init_file
        chmod 755 $init_file
@@ -320,7 +375,7 @@ do
        init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{name}-${service}
        %__cp $yarn_orig_init_file $init_file
        %__sed -i -e 's|@YARN_COMMON_ROOT@|%{lib_hadoop}|' $init_file
-       %__sed -i -e "s|@YARN_DAEMON@|${service}|" $init_file
+       %__sed -i -e "s|@YARN_DAEMON@|${service#yarn-}|" $init_file
        %__sed -i -e 's|@YARN_CONF_DIR@|%{config_hadoop}|' $init_file
        %__sed -i -e 's|@YARN_DAEMON_USER@|yarn|' $init_file
        chmod 755 $init_file
@@ -335,23 +390,36 @@ done
 %__install -d -m 0755 $RPM_BUILD_ROOT/etc/security/limits.d
 %__install -m 0644 %{SOURCE9} $RPM_BUILD_ROOT/etc/security/limits.d/hadoop.nofiles.conf
 
-# /var/lib/hadoop/cache
-%__install -d -m 1777 $RPM_BUILD_ROOT/var/lib/%{name}/cache/hadoop
-# /var/log/hadoop
-%__install -d -m 0755 $RPM_BUILD_ROOT/var/log
-%__install -d -m 0775 $RPM_BUILD_ROOT/var/run/%{name}
-%__install -d -m 0775 $RPM_BUILD_ROOT/var/run/yarn
+# /var/lib/*/cache
+%__install -d -m 1777 $RPM_BUILD_ROOT/%{state_hadoop}/cache
+%__install -d -m 1777 $RPM_BUILD_ROOT/%{state_yarn}/cache
+%__install -d -m 1777 $RPM_BUILD_ROOT/%{state_hdfs}/cache
+%__install -d -m 1777 $RPM_BUILD_ROOT/%{state_mapreduce}/cache
+# /var/log/*
 %__install -d -m 0775 $RPM_BUILD_ROOT/%{log_hadoop}
 %__install -d -m 0775 $RPM_BUILD_ROOT/%{log_yarn}
-
+# %__install -d -m 0775 $RPM_BUILD_ROOT/%{log_hdfs}
+# %__install -d -m 0775 $RPM_BUILD_ROOT/%{log_mapreduce}
+# /var/run/*
+%__install -d -m 0775 $RPM_BUILD_ROOT/%{run_hadoop}
+%__install -d -m 0775 $RPM_BUILD_ROOT/%{run_yarn}
+#%__install -d -m 0775 $RPM_BUILD_ROOT/%{run_hdfs}
+#%__install -d -m 0775 $RPM_BUILD_ROOT/%{run_mapreduce}
 
 %pre
 getent group hadoop >/dev/null || groupadd -r hadoop
+
+%pre hdfs
 getent group hdfs >/dev/null   || groupadd -r hdfs
+getent passwd hdfs >/dev/null || /usr/sbin/useradd --comment "Hadoop HDFS" --shell /bin/bash -M -r -g hdfs -G hadoop --home %{state_hdfs} hdfs
+
+%pre yarn
 getent group yarn >/dev/null   || groupadd -r yarn
+getent passwd yarn >/dev/null || /usr/sbin/useradd --comment "Hadoop Yarn" --shell /bin/bash -M -r -g yarn -G hadoop --home %{state_yarn} yarn
 
-getent passwd hdfs >/dev/null || /usr/sbin/useradd --comment "Hadoop HDFS" --shell /bin/bash -M -r -g hdfs -G hadoop --home %{lib_hadoop} hdfs
-getent passwd yarn >/dev/null || /usr/sbin/useradd --comment "Hadoop Yarn" --shell /bin/bash -M -r -g yarn -G hadoop --home %{lib_hadoop} yarn
+%pre mapreduce
+getent group mapreduce >/dev/null   || groupadd -r mapreduce
+getent passwd mapreduce >/dev/null || /usr/sbin/useradd --comment "Hadoop MapReduce" --shell /bin/bash -M -r -g mapreduce -G hadoop --home %{state_mapreduce} mapreduce
 
 %post
 %{alternatives_cmd} --install %{config_hadoop} %{name}-conf %{etc_hadoop}/conf.empty 10
@@ -362,13 +430,10 @@ getent passwd yarn >/dev/null || /usr/sb
   --slave /etc/%{hadoop_name} %{hadoop_name}-etc %{etc_hadoop} \
   --slave %{man_hadoop}/man1/%{hadoop_name}.1.*z %{hadoop_name}-man %{man_hadoop}/man1/%{name}.1.*z
 
-mkdir -p /var/lib/hadoop/cache/hadoop || :
-chown hdfs:hadoop /var/lib/hadoop/cache/hadoop || :
-chmod g+w /var/lib/hadoop/cache/hadoop/
-mkdir -p /var/log/hadoop || :
-touch /var/log/hadoop/SecurityAuth.audit
-chgrp hadoop /var/log/hadoop/SecurityAuth.audit
-chmod g+w /var/log/hadoop/SecurityAuth.audit
+touch %{log_hadoop}/SecurityAuth.audit
+chgrp hadoop %{log_hadoop}/SecurityAuth.audit
+chmod g+w %{log_hadoop}/SecurityAuth.audit
+
 
 %preun
 if [ "$1" = 0 ]; then
@@ -381,27 +446,94 @@ if [ "$1" = 0 ]; then
   %{alternatives_cmd} --remove %{hadoop_name}-default %{bin_hadoop}/%{name} || :
 fi
 
-%files
+
+%files yarn
 %defattr(-,root,root)
-%config(noreplace) %{etc_hadoop}/conf.empty
-#%config(noreplace) %{etc_yarn}/conf.empty
-%config(noreplace) /etc/default/hadoop
+%config(noreplace) %{etc_hadoop}/conf.empty/yarn-env.sh
+%config(noreplace) %{etc_hadoop}/conf.empty/yarn-site.xml
+%config(noreplace) %{etc_hadoop}/conf.empty/mrapp-generated-classpath
 %config(noreplace) /etc/default/yarn
-%config(noreplace) /etc/security/limits.d/hadoop.nofiles.conf
-%{lib_hadoop}
-%{libexecdir}/hadoop-config.sh
-%{libexecdir}/hdfs-config.sh
-%{libexecdir}/mapred-config.sh
-%{libexecdir}/yarn-config.sh
-%{bin_hadoop}/%{name}
+%{lib_hadoop}/hadoop-yarn*.jar
+%{lib_hadoop}/libexec/yarn-config.sh
+%{lib_hadoop}/sbin/start-yarn.sh
+%{lib_hadoop}/sbin/stop-yarn.sh
+%{lib_hadoop}/sbin/yarn-daemon.sh
+%{lib_hadoop}/sbin/yarn-daemons.sh
+%{lib_hadoop}/bin/yarn
+%{lib_hadoop}/bin/container-executor
 %{bin_hadoop}/yarn
+%attr(0775,yarn,hadoop) %{run_yarn}
+%attr(0775,yarn,hadoop) %{log_yarn}
+%attr(0775,yarn,hadoop) %{state_yarn}
+%attr(1777,yarn,hadoop) %{state_yarn}/cache
+
+%files hdfs
+%defattr(-,root,root)
+%config(noreplace) %{etc_hadoop}/conf.empty/hdfs-site.xml
+%config(noreplace) %{etc_hadoop}/conf.empty/httpfs-*
+%config(noreplace) /etc/default/hadoop-fuse
+%{lib_hadoop}/hadoop-hdfs*.jar
+%{lib_hadoop}/libexec/hdfs-config.sh
+%{lib_hadoop}/libexec/httpfs-config.sh
+%{lib_hadoop}/webapps
+%{lib_hadoop}/sbin/update-hdfs-env.sh
+%{lib_hadoop}/sbin/start-secure-dns.sh
+%{lib_hadoop}/sbin/stop-secure-dns.sh
+%{lib_hadoop}/sbin/start-balancer.sh
+%{lib_hadoop}/sbin/stop-balancer.sh
+%{lib_hadoop}/sbin/start-dfs.sh
+%{lib_hadoop}/sbin/stop-dfs.sh
+%{lib_hadoop}/sbin/refresh-namenodes.sh
+%{lib_hadoop}/sbin/distribute-exclude.sh
+%{lib_hadoop}/sbin/httpfs.sh
+%{lib_hadoop}/bin/hdfs
 %{bin_hadoop}/hdfs
+%attr(0775,hdfs,hadoop) %{run_hdfs}
+%attr(0775,hdfs,hadoop) %{log_hdfs}
+%attr(0775,hdfs,hadoop) %{state_hdfs}
+%attr(1777,hdfs,hadoop) %{state_hdfs}/cache
+
+%files mapreduce
+%defattr(-,root,root)
+%{lib_hadoop}/hadoop-mapreduce*.jar
+%{lib_hadoop}/libexec/mapred-config.sh
+%{lib_hadoop}/bin/mapred
 %{bin_hadoop}/mapred
-%attr(0775,root,hadoop) /var/run/%{name}
-%attr(0775,root,hadoop) %{log_hadoop}
-%attr(0775,root,hadoop) /var/run/yarn
-%attr(0775,root,hadoop) %{log_yarn}
+%attr(0775,mapreduce,hadoop) %{run_mapreduce}
+%attr(0775,mapreduce,hadoop) %{log_mapreduce}
+%attr(0775,mapreduce,hadoop) %{state_mapreduce}
+%attr(1777,mapreduce,hadoop) %{state_mapreduce}/cache
+
+
+%files
+%defattr(-,root,root)
+%config(noreplace) %{etc_hadoop}/conf.empty/hadoop-metrics.properties
+%config(noreplace) %{etc_hadoop}/conf.empty/hadoop-metrics2.properties
+%config(noreplace) %{etc_hadoop}/conf.empty/log4j.properties
+%config(noreplace) %{etc_hadoop}/conf.empty/slaves
+%config(noreplace) %{etc_hadoop}/conf.empty/ssl-client.xml.example
+%config(noreplace) %{etc_hadoop}/conf.empty/ssl-server.xml.example
+%config(noreplace) /etc/default/hadoop
+%config(noreplace) /etc/security/limits.d/hadoop.nofiles.conf
+%{lib_hadoop}/hadoop-common*.jar
+%{lib_hadoop}/hadoop-auth*.jar
+%{lib_hadoop}/hadoop-annotations*.jar
+%{lib_hadoop}/lib
+%{lib_hadoop}/etc
+%{lib_hadoop}/libexec/hadoop-config.sh
+%{lib_hadoop}/sbin/hadoop-*.sh
+%{lib_hadoop}/sbin/update-hadoop-env.sh
+%{lib_hadoop}/sbin/slaves.sh
+%{lib_hadoop}/sbin/start-all.sh
+%{lib_hadoop}/sbin/stop-all.sh
+%{lib_hadoop}/bin/hadoop
+%{lib_hadoop}/bin/rcc
+%{bin_hadoop}/hadoop
 %{man_hadoop}/man1/hadoop.1.*
+%attr(0775,root,hadoop) %{run_hadoop}
+%attr(0775,root,hadoop) %{log_hadoop}
+%attr(0775,root,hadoop) %{state_hadoop}
+%attr(1777,root,hadoop) %{state_hadoop}/cache
 
 %files doc
 %defattr(-,root,root)
@@ -426,12 +558,12 @@ if [ $1 -ge 1 ]; then \
   service %{name}-%1 condrestart >/dev/null 2>&1 \
 fi
 
-%service_macro namenode
-%service_macro secondarynamenode
-%service_macro datanode
-%service_macro resourcemanager
-%service_macro nodemanager
-%service_macro historyserver
+%service_macro hdfs-namenode
+%service_macro hdfs-secondarynamenode
+%service_macro hdfs-datanode
+%service_macro yarn-resourcemanager
+%service_macro yarn-nodemanager
+%service_macro mapreduce-historyserver
 
 # Pseudo-distributed Hadoop installation
 %post conf-pseudo



Mime
View raw message