incubator-ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vik...@apache.org
Subject svn commit: r1346656 - in /incubator/ambari/branches/ambari-186: CHANGES.txt hmc/php/puppet/genmanifest/generateManifest.php hmc/php/puppet/genmanifest/nodeManifest.php hmc/puppet/manifestloader/site.pp hmc/puppet/modules/puppetApply.sh
Date Tue, 05 Jun 2012 23:20:24 GMT
Author: vikram
Date: Tue Jun  5 23:20:23 2012
New Revision: 1346656

URL: http://svn.apache.org/viewvc?rev=1346656&view=rev
Log:
AMBARI-353. Passing packages in hdp class for package install optimization. (Contributed by
Jitendra)

Added:
    incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh
Modified:
    incubator/ambari/branches/ambari-186/CHANGES.txt
    incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php
    incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php
    incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp

Modified: incubator/ambari/branches/ambari-186/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/CHANGES.txt?rev=1346656&r1=1346655&r2=1346656&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/CHANGES.txt (original)
+++ incubator/ambari/branches/ambari-186/CHANGES.txt Tue Jun  5 23:20:23 2012
@@ -6,6 +6,8 @@ characters wide.
 
 Release 0.1.x - unreleased
 
+  AMBARI-353. Passing packages in hdp class for package install optimization. (Jitendra via
Vikram)
+
   AMBARI-344. Fix TxnProgressWidget To Not Hide Previously Pending States (Varun via Vikram)
 
   AMBARI-352. Add flow control - force redirects to appropriate pages based on cluster configuration
status for better usability (Yusaku via Vikram)

Modified: incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php?rev=1346656&r1=1346655&r2=1346656&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php Tue
Jun  5 23:20:23 2012
@@ -60,12 +60,93 @@ include "RoleDependencies.php";
            }
          }
        }
+       ManifestGenerator::optimizePackageInstall($n, $nm, $rolesStates, $configInfo, $roleStages);
        $nodeManifestString = $nm->generateNodeManifest();
        fwrite($fh, $nodeManifestString . "\n");
      }
      fclose($fh);
    }
 
+   static function optimizePackageInstall($node, &$manifestObj, $rolesStatesDs, 
+       $configInfo, $roleStages) {
+     //Figure out the state
+     $serviceState = SERVICE_STATE_INSTALLED_AND_CONFIGURED;
+     foreach ($rolesStatesDs as $r => $stateList) {
+       if ((isset($stateList[SERVICE_STATE_KEY])) &&
+           ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_NO_OP )) {
+         if ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_INSTALLED_AND_CONFIGURED) {
+           $serviceState = NULL;
+           break;
+         }
+       }
+     } 
+     if (!isset($serviceState)) {
+       $serviceState = SERVICE_STATE_UNINSTALLED;
+       //See if it is uninstalled
+       foreach ($rolesStatesDs as $r => $stateList) {
+         if ( (isset($stateList[SERVICE_STATE_KEY])) && 
+           ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_NO_OP )) {
+           if ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_UNINSTALLED) {
+             $serviceState = NULL;
+             break;
+           }
+         }
+       } 
+     }
+     if (!isset($serviceState)) {
+       //No optimization needed
+       return;
+     }
+     //get list of packages
+     $stages = array();
+     foreach($roleStages as $roleName => $val) {
+        $stages[$val] = $roleName;
+     }
+     ksort($stages, SORT_NUMERIC);
+     $packageList = array();
+     foreach($stages as $theStage => $r) {
+       if (!isset($rolesStatesDs[$r])) {
+         continue;
+       }
+       //Add in the order of the stages
+       $stateList = $rolesStatesDs[$r];
+       if ($stateList[SERVICE_STATE_KEY] != $serviceState) { 
+         continue;
+       }
+       if (isset(self::$rolesToPackageMap[$r])) {
+         $p = self::$rolesToPackageMap[$r];
+         if (!in_array($p, $packageList)) {
+           $packageList[] = $p;
+         }
+       }
+     }
+
+     if (empty($packageList)) {
+       //No packages don't bother
+       return;       
+     }
+
+     //lzo and snappy 
+     $packageList[] = self::$rolesToPackageMap["snappy"];
+     if ($configInfo["lzo_enabled"] == "true") {
+       $packageList[] = self::$rolesToPackageMap["lzo"];
+     }
+
+     $firstP = true;
+     $pList = "\"";
+     foreach ($packageList as $p) {
+       if ($firstP) {
+         $firstP = false;
+       } else {
+         $pList = $pList . " ";
+       }
+       $pList = $pList . $p;
+     }
+     $pList = $pList . "\"";
+     $manifestObj->setRoleState("hdp", SERVICE_STATE_KEY, $serviceState);
+     $manifestObj->setRoleState("hdp", "pre_installed_pkgs", $pList);
+   }
+
    private static function getAllImports($modulesDir) {
      $importString = "";
      $importString = $importString . "import \"" . $modulesDir . "/hdp/manifests/*.pp" ."\"\n";
@@ -75,7 +156,6 @@ include "RoleDependencies.php";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-oozie/manifests/*.pp"
."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-pig/manifests/*.pp"
."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-sqoop/manifests/*.pp"
."\"\n";
-     #$importString = $importString . "import \"" . $modulesDir . "/hdp-hcat/manifests/*.pp"
."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-templeton/manifests/*.pp"
."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-hive/manifests/*.pp"
."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-hcat/manifests/*.pp"
."\"\n";
@@ -83,6 +163,36 @@ include "RoleDependencies.php";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-monitor-webserver/manifests/*.pp"
."\"\n";
      return $importString;
    }
+
+   private static $rolesToPackageMap = array (
+     "hdp-hadoop::namenode" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64
hadoop-sbin.x86_64",
+     "hdp-hadoop::snamenode" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64
hadoop-sbin.x86_64",
+     "hdp-hadoop::jobtracker" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64
hadoop-sbin.x86_64",
+     "hdp-hadoop::client" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386
hadoop-sbin.i386",
+     "hdp-hadoop::datanode" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386
hadoop-sbin.i386",
+     "hdp-hadoop::tasktracker" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386
hadoop-sbin.i386",
+     "hdp-hadoop::zookeeper" => "zookeeper",
+     "hdp-hadoop::zookeeper::client" => "zookeeper",
+     "hdp-hbase::master" => "hbase",
+     "hdp-hbase::regionserver" => "hbase",
+     "hdp-hbase::client" => "hbase",
+     "hdp-pig" => "pig.noarch",
+     "hdp-sqoop" => "sqoop",
+     "hdp-hive::server" => "hive",
+     "hdp-hive::client" => "hive",
+     "hdp-hcat" => "hcatalog",
+     "hdp-oozie::server" => "oozie.noarch",
+     "hdp-oozie::client" => "oozie-client.noarch",
+     "hdp-mysql::mysql" => "mysql mysql-server",
+     "hdp-templeton::server" => "templeton.i386",
+     "hdp-templeton::client" => "templeton.i386",
+     "lzo" => "lzo lzo.i386 lzo-devel lzo-devel.i386",
+     "snappy" => "snappy snappy-devel",
+     "hdp-ganglia::monitor_and_server" => "ganglia-gmetad-3.2.0 ganglia-gmond-3.2.0 gweb
hdp_mon_ganglia_addons",
+     "hdp-ganglia::monitor" => "ganglia-gmond-3.2.0 gweb hdp_mon_ganglia_addons",
+     "hdp-nagios::server" => "hdp_mon_nagios_addons nagios-3.2.3 nagios-plugins-1.4.9
fping net-snmp-utils",
+     "hdp-dashboard" => "hdp_mon_dashboard",
+   );
  }
 
 ?>

Modified: incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php?rev=1346656&r1=1346655&r2=1346656&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php Tue Jun
 5 23:20:23 2012
@@ -186,20 +186,4 @@
     }
   }
 
-/*** TEST CODE ********
-  $nm = new NodeManifest("x.y.z");
-  $nm->setRoleStage("namenode", "1");
-  $nm->setRoleStage("datanode", "2");
-  $nm->setRoleStage("namenode", "3");
-  $nm->setRoleState("namenode", "service_state", "running");
-  $nm->setRoleOpt("namenode", "deploy", "yes");
-  $nm->setRoleOpt("namenode", "deploy", "no");
-  $nm->setRoleOpt("namenode", "wait", "10");
-  $nm->setRoleOpt("datanode", "running", "yes");
-  $nm->setNamenodeRoleState("stopped", 4);
-  $nm->setDatanodeRoleState("running", 5);
-  $ret = $nm->generateNodeManifest();
-  echo "$ret\n";
-***/
-
 ?>

Modified: incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp?rev=1346656&r1=1346655&r2=1346656&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp (original)
+++ incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp Tue Jun  5 23:20:23
2012
@@ -1,38 +1,23 @@
 class manifestloader () {
-    anchor { 'manifestloader::begin': }
     file { '/etc/puppet/agent/site.pp':
       ensure => present,
       source => "puppet:///modules/catalog/site.pp",  
       mode => '0755',
-      require => Anchor['manifestloader::begin'],      
-      notify => Exec['rm_puppet_apply_log']
     }
 
     exec {'rm_puppet_apply_log':
       command   => "rm -f /var/log/puppet_apply.log",
       path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-      notify    => Exec['puppet_apply'],
-      require   => File['/etc/puppet/agent/site.pp'],
-      logoutput => "true"
     }
 
     exec { 'puppet_apply':
-      command   => "puppet apply --confdir=/etc/puppet/agent --logdest=/var/log/puppet_apply.log
--debug --autoflush /etc/puppet/agent/site.pp",
+      command   => "sh /etc/puppet/agent/modules/puppetApply.sh",
       timeout   => 1200,
       path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-      require   => Exec['rm_puppet_apply_log'],
-      notify    => Exec['append'],
       logoutput => "true"
     }
 
-    exec { 'append':
-      command   => "cat /var/log/puppet_apply.log",
-      path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-      require   => Exec['puppet_apply'],
-      notify   => Anchor['manifestloader::end'],
-      logoutput => "true"
-    }
-    anchor { 'manifestloader::end': }
+    File['/etc/puppet/agent/site.pp'] -> Exec['rm_puppet_apply_log'] -> Exec['puppet_apply']
 }
 
 node default {

Added: incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh?rev=1346656&view=auto
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh (added)
+++ incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh Tue Jun  5 23:20:23
2012
@@ -0,0 +1,7 @@
+puppet apply --confdir=/etc/puppet/agent --logdest=console --debug --autoflush --detailed-exitcodes
/etc/puppet/agent/site.pp
+ret=$?
+if [ "$ret" == "0" ] || [ "$ret" == "2" ]; then
+  exit 0
+else
+  exit 1 
+fi



Mime
View raw message