hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r608950 [1/7] - in /lucene/hadoop/trunk/src: contrib/hod/ contrib/hod/bin/ contrib/hod/conf/ contrib/hod/hodlib/ contrib/hod/hodlib/AllocationManagers/ contrib/hod/hodlib/Common/ contrib/hod/hodlib/GridServices/ contrib/hod/hodlib/Hod/ cont...
Date Fri, 04 Jan 2008 18:20:22 GMT
Author: nigel
Date: Fri Jan  4 10:20:17 2008
New Revision: 608950

URL: http://svn.apache.org/viewvc?rev=608950&view=rev
Log:
HADOOP-1301.  Hadoop-On-Demand (HOD): resource management provisioning for Hadoop. Contributed by Hemanth Yamijala.

Added:
    lucene/hadoop/trunk/src/contrib/hod/
    lucene/hadoop/trunk/src/contrib/hod/README
    lucene/hadoop/trunk/src/contrib/hod/bin/
    lucene/hadoop/trunk/src/contrib/hod/bin/VERSION
    lucene/hadoop/trunk/src/contrib/hod/bin/hod
    lucene/hadoop/trunk/src/contrib/hod/bin/hodring
    lucene/hadoop/trunk/src/contrib/hod/bin/ringmaster
    lucene/hadoop/trunk/src/contrib/hod/build.xml
    lucene/hadoop/trunk/src/contrib/hod/conf/
    lucene/hadoop/trunk/src/contrib/hod/conf/hodrc
    lucene/hadoop/trunk/src/contrib/hod/config.txt
    lucene/hadoop/trunk/src/contrib/hod/getting_started.txt
    lucene/hadoop/trunk/src/contrib/hod/hodlib/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/goldAllocationManager.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/allocationManagerUtil.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/desc.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/descGenerator.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/hodsvc.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/logger.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/miniHTMLParser.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/nodepoolutil.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/setup.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/socketServers.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/tcp.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/threads.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/types.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/util.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/xmlrpc.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/GridServices/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/GridServices/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/GridServices/mapred.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/GridServices/service.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Hod/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Hod/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Hod/hadoop.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Hod/hod.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Hod/nodePool.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/HodRing/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/HodRing/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/HodRing/hodRing.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/NodePools/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/NodePools/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/NodePools/torque.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/RingMaster/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/RingMaster/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/RingMaster/idleJobTracker.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/RingMaster/ringMaster.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Schedulers/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Schedulers/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/Schedulers/torque.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/ServiceProxy/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/ServiceProxy/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/ServiceProxy/serviceProxy.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/ServiceRegistry/
    lucene/hadoop/trunk/src/contrib/hod/hodlib/ServiceRegistry/__init__.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/ServiceRegistry/serviceRegistry.py   (with props)
    lucene/hadoop/trunk/src/contrib/hod/hodlib/__init__.py   (with props)
    lucene/hadoop/trunk/src/docs/src/documentation/content/xdocs/hod.xml

Added: lucene/hadoop/trunk/src/contrib/hod/README
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/README?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/README (added)
+++ lucene/hadoop/trunk/src/contrib/hod/README Fri Jan  4 10:20:17 2008
@@ -0,0 +1,104 @@
+                        Hadoop On Demand
+                        ================
+
+1. Introduction:
+================
+
+The Hadoop On Demand (HOD) project is a system for provisioning and 
+managing independent Hadoop MapReduce instances on a shared cluster 
+of nodes. HOD uses a resource manager for allocation. At present it
+supports Torque (http://www.clusterresources.com/pages/products/torque-resource-manager.php)
+out of the box. 
+
+2. Feature List:
+================
+
+The following are the features provided by HOD:
+
+2.1 Simplified interface for managing MapReduce clusters:
+
+The MapReduce user interacts with the cluster through a simple 
+command line interface, the HOD client. HOD brings up a virtual 
+MapReduce cluster with the required number of nodes, which the 
+user can use for running Hadoop jobs. When done, HOD will 
+automatically clean up the resources and make the nodes available 
+again.
+
+2.2 Automatic installation of Hadoop:
+
+With HOD, Hadoop does not need to be even installed on the cluster.
+The user can provide a Hadoop tarball that HOD will automatically 
+distribute to all the nodes in the cluster.
+
+2.3 Configuring Hadoop:
+
+Dynamic parameters of Hadoop configuration, such as the NameNode and 
+JobTracker addresses and ports, and file system temporary directories
+are generated and distributed by HOD automatically to all nodes in
+the cluster.
+
+In addition, HOD allows the user to configure Hadoop parameters
+at both the server (for e.g. JobTracker) and client (for e.g. JobClient)
+level, including 'final' parameters, that were introduced with 
+Hadoop 0.15.
+
+2.4 Auto-cleanup of unused clusters:
+
+HOD has an automatic timeout so that users cannot misuse resources they 
+aren't using. The timeout applies only when there is no MapReduce job 
+running. 
+
+2.5 Log services:
+
+HOD can be used to collect all MapReduce logs to a central location
+for archiving and inspection after the job is completed.
+
+3. HOD Components
+=================
+
+This is a brief overview of the various components of HOD and how they
+interact to provision Hadoop.
+
+HOD Client: The HOD client is a Unix command that users use to allocate 
+Hadoop MapReduce clusters. The command provides other options to list 
+allocated clusters and deallocate them. The HOD client generates the 
+hadoop-site.xml in a user specified directory. The user can point to 
+this configuration file while running Map/Reduce jobs on the allocated 
+cluster.
+
+RingMaster: The RingMaster is a HOD process that is started on one node 
+per every allocated cluster. It is submitted as a 'job' to the resource 
+manager by the HOD client. It controls which Hadoop daemons start on 
+which nodes. It provides this information to other HOD processes, 
+such as the HOD client, so users can also determine this information. 
+The RingMaster is responsible for hosting and distributing the 
+Hadoop tarball to all nodes in the cluster. It also automatically 
+cleans up unused clusters.
+
+HodRing: The HodRing is a HOD process that runs on every allocated node
+in the cluster. These processes are run by the RingMaster through the 
+resource manager, using a facility of parallel execution. The HodRings
+are responsible for launching Hadoop commands on the nodes to bring up 
+the Hadoop daemons. They get the command to launch from the RingMaster.
+
+Hodrc / HOD configuration file: An INI style configuration file where
+the users configure various options for the HOD system, including
+install locations of different software, resource manager parameters,
+log and temp file directories, parameters for their MapReduce jobs,
+etc.
+
+Submit Nodes: Nodes where the HOD Client is run, from where jobs are
+submitted to the resource manager system for allocating and running 
+clusters.
+
+Compute Nodes: Nodes which get allocated by a resource manager, 
+and on which the Hadoop daemons are provisioned and started.
+
+4. Next Steps:
+==============
+
+- Read getting_started.txt to get an idea of how to get started with
+installing, configuring and running HOD.
+
+- Read config.txt to get more details on configuration options for HOD.
+

Added: lucene/hadoop/trunk/src/contrib/hod/bin/VERSION
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/bin/VERSION?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/bin/VERSION (added)
+++ lucene/hadoop/trunk/src/contrib/hod/bin/VERSION Fri Jan  4 10:20:17 2008
@@ -0,0 +1,16 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+DEVELOPMENT

Added: lucene/hadoop/trunk/src/contrib/hod/bin/hod
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/bin/hod?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/bin/hod (added)
+++ lucene/hadoop/trunk/src/contrib/hod/bin/hod Fri Jan  4 10:20:17 2008
@@ -0,0 +1,484 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#!/bin/sh
+""":"
+work_dir=$(dirname $0)
+base_name=$(basename $0)
+original_dir=$PWD
+cd $work_dir
+
+if [ $HOD_PYTHON_HOME ]; then
+    exec $HOD_PYTHON_HOME -u -OO $base_name ${1+"$@"} --hod.original-dir $original_dir
+elif [ -e /usr/bin/python ]; then
+    exec /usr/bin/python -u -OO $base_name ${1+"$@"} --hod.original-dir $original_dir
+elif [ -e /usr/local/bin/python ]; then
+    exec /usr/local/bin/python -u -OO $base_name ${1+"$@"} --hod.original-dir $original_dir
+else
+    exec python -u -OO $base_name ${1+"$@"} --hod.original-dir $work_dir
+fi
+":"""
+
+"""The executable to be used by the user"""
+import sys, os, re, pwd, threading, sys
+
+myName          = os.path.basename(sys.argv[0])
+myName          = re.sub(".*/", "", myName)
+binDirectory    = os.path.realpath(sys.argv[0])
+rootDirectory   = re.sub("/bin/.*", "", binDirectory)
+libDirectory    = rootDirectory
+
+sys.path.append(libDirectory)
+
+from hodlib.Hod.hod import hodRunner
+from hodlib.Common.setup import *
+from hodlib.Common.descGenerator import *
+from hodlib.Common.util import local_fqdn, need_to_allocate, filter_warnings, get_exception_error_string
+from hodlib.Common.tcp import tcpError, tcpSocket
+
+filter_warnings()
+
+reVersion = re.compile(".*(\d+_\d+).*")
+
+VERSION = None
+if os.path.exists("./VERSION"):
+  vFile = open("./VERSION", 'r')
+  VERSION = vFile.readline()
+  vFile.close()
+
+# Always look for hodrc file here unless otherwise specified with -c:   
+DEFAULT_LOC = os.path.join(rootDirectory, 'conf')
+DEFAULT_HOD_DIR = os.path.join(os.environ['HOME'], ".hod")
+
+if not os.path.isdir(DEFAULT_HOD_DIR):
+  os.mkdir(DEFAULT_HOD_DIR, 0777)
+
+DEFAULT_CONFIG = os.path.join(DEFAULT_HOD_DIR, 'hodrc')
+if not os.path.exists(DEFAULT_CONFIG):
+  if os.environ.has_key('HOD_CONF_DIR') and os.environ['HOD_CONF_DIR'] is not None:
+    DEFAULT_CONFIG = os.path.join(os.environ['HOD_CONF_DIR'], 'hodrc')
+
+# Definition tuple is of the form:
+#  (name, type, description, help?, default value, required?, validate?, 
+#   short option)
+#
+defList = { 'hod' : (      
+             ('original-dir', 'directory', 'hod original start directory',
+              False, None, True, True, 'r'),        
+
+             ('syslog-address', 'address', 'Syslog address.',
+              False, None, False, True, 'y'),
+              
+             ('java-home', 'directory', 'Java home directory.',
+              True, None, True, True, 'j'),
+            
+             ('debug', 'pos_int', 'Debugging level, 0-4.',
+              True, 3, True, True, 'b'),
+            
+             ('stream', 'bool', 'Output to stderr.',
+              False, True, False, True, 's'),
+
+             ('min-nodes', 'pos_int', 
+              'Minimum number of nodes to allocate at startup.',
+              True, None, False, True, 'm'),
+
+             ('script', 'file', 'Hadoop script to execute.',
+              True, None, False, True, 'z'), 
+
+             ('userid', 'user_account', 
+              'User ID the hod shell is running under.',
+              False, pwd.getpwuid(os.getuid())[0], False, True, 'u'),
+             
+             ('allocate-wait-time', 'pos_int', 
+              'Time to wait for cluster allocation.',
+              False, 300, True, True, 'e'),         
+             
+             ('operation', 'string', 
+              'Initiate a hod operation. (help, allocate, deallocate ...)',
+              True, None, False, True, 'o'),
+              
+             ('cluster-factor', 'pos_float',
+              'The number of grid slots per machines', False, 1.9, False, True,
+              'x'),
+             
+             ('cluster', 'string', 'Name of cluster being used.',
+              False, None, True, True, 'w'),
+
+             ('proxy-xrs-address', 'address', 
+              'Address to Allocation Manager XML RPC proxy.',
+              False, None, False, True, 'p'),
+              
+             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
+              False, None, True, True),
+
+             ('client-params', 'keyval', 'Hadoop client xml key/value list',
+              False, None, False, True, 'C'), 
+
+             ('hadoop-ui-log-dir', 'directory', 'Directory to store Web UI Logs of Hadoop',
+              False, None, False, True)),
+
+            'resource_manager' : (
+             ('id', 'string', 'Batch scheduler ID: torque|condor.',
+              False, None, True, True),
+             
+             ('pbs-user', 'user_account', 'User ID jobs are submitted under.',
+              False, None, False, True),
+              
+             ('pbs-account', 'string', 'User Account jobs are submitted under.',
+              True, pwd.getpwuid(os.getuid())[0], False, False, 'A'),
+              
+             ('queue', 'string', 'Queue of the batch scheduler to query.',
+              True, 'batch', False, True, 'Q'),
+             
+             ('batch-home', 'directory', 'Scheduler installation directory.',
+              False, None, True, True),
+             
+             ('options', 'keyval', 'Options to pass to the scheduler.',
+              False, None, False, True),
+
+             ('env-vars', 'keyval', 'Environment variables to pass to the submitted jobs.',
+              False, None, False, True)),
+                            
+            'ringmaster' : (
+             ('work-dirs', 'list', 'hod work directories',
+              False, None, True, False),
+
+             ('temp-dir', 'directory', 'Ringmaster temporary directory.',
+              False, None, True, False),
+              
+             ('log-dir', 'directory', 'hod logging directory.', 
+              False, os.path.join(rootDirectory, 'logs'), False, False),
+
+             ('syslog-address', 'address', 'Syslog address.',
+              False, None, False, False),
+
+             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
+              False, None, True, True),
+              
+             ('http-port-range', 'range', 'HTTP port range n-m.',
+              False, None, True, True),
+              
+             ('debug', 'pos_int', 'Debugging level, 0-4.',
+              False, 4, True,   True),
+               
+             ('register', 'bool', 'Register with service registry?',
+              False, True, True, True),
+               
+             ('stream', 'bool', 'Output to stderr.',
+              False, False, False, True),
+              
+             ('userid', 'user_account', 
+              'User ID the hod shell is running under.',
+              False, pwd.getpwuid(os.getuid())[0], False, True),
+               
+             ('svcrgy-addr', 'address', 'Download HTTP address.',
+              False, None, False, False),             
+             
+             ('hadoop-tar-ball', 'string', 'hadoop program tar ball.',
+              True, None, False, False, 't'),
+
+             ('max-connect','pos_int','max connections allowed for a single tarball server',
+             False, 30, False, True),
+
+             ('jt-poll-interval', 'pos_int', 'How often to poll the Job tracker for idleness',
+             False, 120, False, True),
+
+             ('idleness-limit', 'pos_int', 'Limit after which to deallocate the cluster',
+             False, 3600, False, True)),
+
+            'gridservice-mapred' : (
+             ('external', 'bool', "Connect to an already running MapRed?",
+              False, False, True, True),
+              
+             ('host', 'hostname', 'Mapred hostname.', 
+              False, 'localhost', False, True),
+
+             ('info_port', 'pos_int', 'Mapred info port.',
+              False, None, True, True), 
+             
+             ('tracker_port', 'pos_int', 'Mapred job tracker port.',
+              False, None, True, True),
+                        
+             ('cmdline-params', 'keyval', 'Hadoop cmdline key/value list.',
+              False, None, False, False),
+
+             ('server-params', 'keyval', 'Hadoop xml key/value list',
+              False, None, False, True, 'M'),
+               
+             ('envs', 'keyval', 'environment to run this package in',
+              False, None, False, False),
+
+             ('final-server-params', 'keyval', 'Hadoop final xml key/val list',
+              False, None, False, True, 'F'),
+
+             ('java-opts', 'list', 'java jvm options',
+              False, None, False, False),
+               
+             ('pkgs', 'directory', "directory where the package is installed",
+              False, None, False, False)), 
+               
+               
+            'gridservice-hdfs' : (
+             ('external', 'bool', "Connect to an already running HDFS?",
+              False, False, True, True),
+             
+             ('host', 'hostname', 'HDFS hostname.', 
+              False, 'localhost', False, True),
+             
+             ('fs_port', 'pos_int', 'HDFS port.',
+              False, None, True, True),
+              
+             ('info_port', 'pos_int', 'HDFS info port.',
+              False, None, True, True), 
+             
+             ('cmdline-params', 'keyval', 'Hadoop cmdline key/value list.',
+              False, None, False, False),
+
+             ('server-params', 'keyval', 'Hadoop xml key/value list',
+              False, None, False, True, 'H'),
+
+             ('final-server-params', 'keyval', 'Hadoop final xml key/value list',
+              False, None, False, True, 'S'),
+           
+             ('envs', 'keyval', 'Environment in which to run this package.',
+              False, None, False, False),
+
+             ('java-opts', 'list', 'java jvm options',
+              False, None, False, False),
+            
+             ('pkgs', 'directory', "directory where the package is installed",
+              False, None, False, False)),           
+             
+             
+            'hodring' : (
+             ('temp-dir', 'list', 'hodring temporary directory.',
+              False, None, True, False),
+              
+             ('log-dir', 'directory', 'hod logging directory.', 
+              False, os.path.join(rootDirectory, 'logs'), False, False), 
+              
+             ('log-destination-uri', 'string', 
+              'URI to store logs to, local://some_path or '
+              + 'hdfs://host:port/some_path', 
+              False, None, False, True),
+
+             ('pkgs', 'directory', 'Path to Hadoop to use in case of uploading to HDFS',
+              False, None, False, False),
+              
+             ('syslog-address', 'address', 'Syslog address.',
+              False, None, False, False),
+          
+             ('java-home', 'directory', 'Java home directory.',
+              False, None, True, False),
+              
+             ('debug', 'pos_int', 'Debugging level, 0-4.',
+              False, 3, True, True),
+               
+             ('register', 'bool', 'Register with service registry?',
+              False, True, True, True),
+               
+             ('stream', 'bool', 'Output to stderr.',
+              False, False, False, True),
+
+             ('userid', 'user_account', 
+              'User ID the hod shell is running under.',
+              False, pwd.getpwuid(os.getuid())[0], False, True),
+               
+             ('command', 'string', 'Command for hodring to run.',
+              False, None, False, True),
+
+             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
+              False, None, True, True),
+               
+             ('http-port-range', 'range', 'HTTP port range n-m.',
+              False, None, True, True),
+              
+             ('service-id', 'string', 'Service ID.',
+              False, None, False, True),
+              
+             ('download-addr', 'string', 'Download HTTP address.',
+              False, None, False, True),
+               
+             ('svcrgy-addr', 'address', 'Download HTTP address.',
+              False, None, False, True), 
+    
+             ('ringmaster-xrs-addr', 'address', 'Ringmaster XML-RPC address.',
+              False, None, False, True),
+
+             ('tarball-retry-initial-time', 'pos_float','Initial Retry time for tarball download',
+              False, 1, False, True),
+              
+             ('tarball-retry-interval', 'pos_float','interval to spread retries for tarball download',
+              False, 3, False, True),
+              
+             ('cmd-retry-initial-time', 'pos_float','Initial retry time for getting commands',
+              False, 2, False, True),
+             
+             ('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
+              False, 2, False, True)) 
+              }   
+
+defOrder = [ 'hod', 'ringmaster', 'hodring', 'resource_manager', 
+             'gridservice-mapred', 'gridservice-hdfs' ]
+
+def printErrors(msgs):
+  for msg in msgs:
+    print msg
+
+def op_requires_pkgs(config):
+  if config['hod'].has_key('operation'):
+    return config['hod']['operation'].startswith('allocate')
+  else:
+    return config['hod'].has_key('script')
+
+if __name__ == '__main__':  
+  confDef = definition()
+  confDef.add_defs(defList, defOrder)
+  hodOptions = options(confDef, "./%s -c <CONFIG_FILE> [OPTIONS]" % myName,
+                       VERSION, withConfig=True, defaultConfig=DEFAULT_CONFIG)
+
+  # hodConfig is a dict like object, hodConfig[section][name]
+  try:
+    hodConfig = config(hodOptions['config'], configDef=confDef, 
+                     originalDir=hodOptions['hod']['original-dir'],
+                     options=hodOptions) 
+  except IOError, e:
+    print >>sys.stderr,"error: %s not found. Specify the path to the HOD configuration file, or define the environment variable %s under which a file named hodrc can be found." % (hodOptions['config'], 'HOD_CONF_DIR')
+    sys.exit(1)
+
+  status = True
+  statusMsgs = []
+
+  (status,statusMsgs) = hodConfig.verify()
+  if not status:
+    print >>sys.stderr,"error: bin/hod failed to start."
+    for msg in statusMsgs:
+      print >>sys.stderr,"%s" % (msg)
+    sys.exit(1)
+
+  ## TODO : should move the dependency verification to hodConfig.verify
+  if hodConfig['hod'].has_key('script') \
+    and not hodConfig['hod'].has_key('min-nodes'):
+    printErrors(hodConfig.var_error('hod', 'min-nodes',
+        "hod.min-nodes must be specified when using hod.script option."))
+    sys.exit(1)
+
+  if hodConfig['hod'].has_key('min-nodes'):
+    if hodConfig['hod']['min-nodes'] < 3:
+      printErrors(hodConfig.var_error('hod', 'min-nodes',
+        "hod.min-nodes must be >= 3 nodes: %s." % 
+        hodConfig['hod']['min-nodes']))
+      sys.exit(1)
+  
+  if hodConfig['hod'].has_key('operation') and \
+    hodConfig['hod'].has_key('script'):
+    print "Script execution and hod operations are mutually exclusive."
+    hodOptions.print_help(sys.stderr)
+    sys.exit(1)
+  
+  if 'operation' not in hodConfig['hod'] and 'script' not in hodConfig['hod']:
+    print "HOD requires at least a script or operation be specified."
+    hodOptions.print_help(sys.stderr)
+    sys.exit(1)    
+  
+  if hodConfig['gridservice-hdfs']['external']:
+    hdfsAddress = "%s:%s" % (hodConfig['gridservice-hdfs']['host'], 
+                             hodConfig['gridservice-hdfs']['fs_port'])
+
+    hdfsSocket = tcpSocket(hdfsAddress)
+      
+    try:
+      hdfsSocket.open()
+      hdfsSocket.close()
+    except tcpError:
+      printErrors(hodConfig.var_error('hod', 'gridservice-hdfs', 
+        "Failed to open a connection to external hdfs address: %s." % 
+        hdfsAddress))
+      sys.exit(1)
+  else:
+    hodConfig['gridservice-hdfs']['host'] = 'localhost'
+
+  if hodConfig['gridservice-mapred']['external']:
+    mapredAddress = "%s:%s" % (hodConfig['gridservice-mapred']['host'], 
+                               hodConfig['gridservice-mapred']['tracker_port'])
+
+    mapredSocket = tcpSocket(mapredAddress)
+      
+    try:
+      mapredSocket.open()
+      mapredSocket.close()
+    except tcpError:
+      printErrors(hodConfig.var_error('hod', 'gridservice-mapred', 
+        "Failed to open a connection to external mapred address: %s." % 
+        mapredAddress))
+      sys.exit(1)
+  else:
+    hodConfig['gridservice-mapred']['host'] = 'localhost'
+
+  if not hodConfig['ringmaster'].has_key('hadoop-tar-ball') and \
+    not hodConfig['gridservice-hdfs'].has_key('pkgs') and \
+    op_requires_pkgs(hodConfig):
+    printErrors(hodConfig.var_error('gridservice-hdfs', 'pkgs', 
+      "gridservice-hdfs.pkgs must be defined if ringmaster.hadoop-tar-ball "
+      + "is not defined."))
+    sys.exit(1)
+
+  if not hodConfig['ringmaster'].has_key('hadoop-tar-ball') and \
+    not hodConfig['gridservice-mapred'].has_key('pkgs') and \
+    op_requires_pkgs(hodConfig):
+    printErrors(hodConfig.var_error('gridservice-mapred', 'pkgs', 
+      "gridservice-mapred.pkgs must be defined if ringmaster.hadoop-tar-ball "
+      + "is not defined."))
+    sys.exit(1)
+
+  if hodConfig['hodring'].has_key('log-destination-uri'):
+    if hodConfig['hodring']['log-destination-uri'].startswith('file://'):
+      pass
+    elif hodConfig['hodring']['log-destination-uri'].startswith('hdfs://'):
+      hostPort = hodConfig['hodring']['log-destination-uri'][7:].split("/")
+      hostPort = hostPort[0]
+      socket = tcpSocket(hostPort)
+      try:
+        socket.open()
+        socket.close()
+      except:
+        printErrors(hodConfig.var_error('hodring', 'log-destination-uri', 
+        "Unable to contact host/port specified in log destination uri: %s" % 
+        hodConfig['hodring']['log-destination-uri']))
+        sys.exit(1)
+    else:
+      printErrors(hodConfig.var_error('hodring', 'log-destination-uri', 
+        "The log destiniation uri must be of type local:// or hdfs://."))
+      sys.exit(1)
+  ## TODO : end of should move the dependency verification to hodConfig.verif
+    
+  hodConfig['hod']['base-dir'] = rootDirectory
+  hodConfig['hod']['user_state'] = DEFAULT_HOD_DIR
+
+  dGen = DescGenerator(hodConfig)
+  hodConfig = dGen.initializeDesc()
+  
+  os.environ['JAVA_HOME'] = hodConfig['hod']['java-home']
+  
+  if hodConfig['hod']['debug'] == 4:
+    print ""
+    print "Using Python: %s" % sys.version
+    print ""
+  
+  hod = hodRunner(hodConfig)
+  if hodConfig['hod'].has_key('script'):
+    sys.exit(hod.script())
+  else:  
+    sys.exit(hod.operation())

Added: lucene/hadoop/trunk/src/contrib/hod/bin/hodring
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/bin/hodring?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/bin/hodring (added)
+++ lucene/hadoop/trunk/src/contrib/hod/bin/hodring Fri Jan  4 10:20:17 2008
@@ -0,0 +1,198 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#!/bin/sh
+""":"
+work_dir=$(dirname $0)
+base_name=$(basename $0)
+cd $work_dir
+
+if [ $HOD_PYTHON_HOME ]; then
+    exec $HOD_PYTHON_HOME -OO $base_name ${1+"$@"}
+elif [ -e /usr/bin/python ]; then
+    exec /usr/bin/python -OO $base_name ${1+"$@"}
+elif [ -e /usr/local/bin/python ]; then
+    exec /usr/local/bin/python -OO $base_name ${1+"$@"}
+else
+    exec python -OO $base_name ${1+"$@"}
+fi
+":"""
+
+"""The executable to be used by the user"""
+import sys, os, re
+
+
+myName          = os.path.basename(sys.argv[0])
+myName          = re.sub(".*/", "", myName)
+binDirectory    = os.path.realpath(sys.argv[0])
+rootDirectory   = re.sub("/bin/.*", "", binDirectory)
+libDirectory    = rootDirectory
+
+sys.path.append(libDirectory)
+
+from hodlib.HodRing.hodRing import HodRing
+from hodlib.Common.setup import *
+from hodlib.Common.util import filter_warnings,get_exception_string, get_exception_error_string
+from hodlib.Common.logger import hodLog
+
+filter_warnings()
+
+reVersion = re.compile(".*(\d+_\d+).*")
+
+VERSION = '$HeadURL$'
+
+reMatch = reVersion.match(VERSION)
+if reMatch:
+    VERSION = reMatch.group(1)
+    VERSION = re.sub("_", ".", VERSION)
+else:
+    VERSION = 'DEV'
+
+# Definition tuple is of the form:
+#  (name, type, description, default value, required?, validate?)
+#
+defList = { 'hodring' : (
+             ('temp-dir', 'directory', 'hod work directories',
+              False, None, True, False),
+              
+             ('log-dir', 'directory', 'hod logging directory.', 
+              False, os.path.join(rootDirectory, 'logs'), False, True), 
+
+             ('log-destination-uri', 'string', 
+              'URI to store logs to, local://some_path or '
+              + 'hdfs://host:port/some_path', 
+              False, None, False, True), 
+
+             ('pkgs', 'directory', 'Path to Hadoop to use in case of uploading to HDFS',
+              False, None, False, True),
+              
+             ('syslog-address', 'address', 'Syslog address.',
+              False, None, False, True),
+          
+             ('java-home', 'directory', 'Java home directory.',
+              False, None, True, True),
+              
+             ('debug', 'pos_int', 'Debugging level, 0-4.',
+              False, 3, True, True),
+               
+             ('register', 'bool', 'Register with service registry?',
+              False, True, True, True),
+               
+             ('stream', 'bool', 'Output to stderr.',
+              False, False, False, True),
+
+             ('userid', 'user_account', 
+              'User ID the hod shell is running under.',
+              False, None, True, False),
+
+             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
+              False, None, True, True),
+               
+             ('http-port-range', 'range', 'HTTP port range n-m.',
+              False, None, True, True),
+               
+             ('command', 'string', 'Command for hodring to run.',
+              False, None, False, True),
+              
+             ('service-id', 'string', 'Service ID.',
+              False, None, False, True),
+              
+             ('download-addr', 'string', 'Download HTTP address.',
+              False, None, False, True),
+               
+             ('svcrgy-addr', 'address', 'Service registry XMLRPC address.',
+              False, None, True, True), 
+    
+             ('ringmaster-xrs-addr', 'address', 'Ringmaster XML-RPC address.',
+              False, None, False, True),
+ 
+             ('tarball-retry-initial-time', 'pos_float','initial retry time for tarball download',
+              False, 1, False, True),
+              
+             ('tarball-retry-interval', 'pos_float','interval to spread retries for tarball download',
+              False, 3, False, True),
+              
+             ('cmd-retry-initial-time', 'pos_float','initial retry time for getting commands',
+              False, 2, False, True),
+             
+             ('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
+              False, 2, False, True)) 
+            }            
+
+def getLogger(hodRingOptions):
+  if hodRingOptions['hodring']['debug'] > 0:
+    _baseLogger = hodLog('hodring')
+    log = _baseLogger.add_logger('main')
+
+    if hodRingOptions['hodring'].has_key('stream'):
+      if hodRingOptions['hodring']['stream']:
+        _baseLogger.add_stream(level=hodRingOptions['hodring']['debug'],
+                             addToLoggerNames=('main',))
+
+    _serviceID = os.getenv('PBS_JOBID')
+
+    if hodRingOptions['hodring'].has_key('log-dir'):
+      if _serviceID:
+          __logDir = os.path.join(hodRingOptions['hodring']['log-dir'], "%s.%s" % (
+                                   hodRingOptions['hodring']['userid'], _serviceID))
+      else:
+          __logDir = os.path.join(hodRingOptions['hodring']['log-dir'],
+                                       hodRingOptions['hodring']['userid'])
+      if not os.path.exists(__logDir):
+        os.mkdir(__logDir)
+
+      _baseLogger.add_file(logDirectory=__logDir,
+        level=hodRingOptions['hodring']['debug'], addToLoggerNames=('main',))
+
+    if hodRingOptions['hodring'].has_key('syslog-address'):
+      _baseLogger.add_syslog(hodRingOptions['hodring']['syslog-address'],
+        level=hodRingOptions['hodring']['debug'], addToLoggerNames=('main',))
+    return log
+
+def ensureLogDir(logDir):
+  """Verify that the passed in log directory exists, and if it doesn't
+  create it."""
+  if not os.path.exists(logDir):
+    try:
+      old_mask = os.umask(0)
+      os.makedirs(logDir, 01777)
+      os.umask(old_mask)
+    except Exception, e:
+      print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
+      raise e
+
+if __name__ == '__main__':
+
+  confDef = definition()
+  confDef.add_defs(defList)
+  hodRingOptions = options(confDef, "./%s [OPTIONS]" % myName, VERSION)
+  ensureLogDir(hodRingOptions['hodring']['log-dir'])
+  service = None
+  try:
+    (status, statusMsgs) = hodRingOptions.verify()
+    if not status:
+      raise Exception("%s" % statusMsgs)
+    hodRingOptions['hodring']['base-dir'] = rootDirectory
+    service = HodRing(hodRingOptions)
+    service.start()
+    service.wait()
+
+  except Exception:
+    if service:
+      if service.log:
+        log = service.log
+    else:
+      log = getLogger(hodRingOptions)
+    log.error("bin/hodring failed to start. %s. \nStack trace:\n%s" %(get_exception_error_string(),get_exception_string()))

Added: lucene/hadoop/trunk/src/contrib/hod/bin/ringmaster
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/bin/ringmaster?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/bin/ringmaster (added)
+++ lucene/hadoop/trunk/src/contrib/hod/bin/ringmaster Fri Jan  4 10:20:17 2008
@@ -0,0 +1,330 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+#!/bin/sh
+""":"
+work_dir=$(dirname $0)
+base_name=$(basename $0)
+cd $work_dir
+
+if [ $HOD_PYTHON_HOME ]; then
+    exec $HOD_PYTHON_HOME -OO $base_name ${1+"$@"}
+elif [ -e /usr/bin/python ]; then
+    exec /usr/bin/python -OO $base_name ${1+"$@"}
+elif [ -e /usr/local/bin/python ]; then
+    exec /usr/local/bin/python -OO $base_name ${1+"$@"}
+else
+    exec python -OO $base_name ${1+"$@"}
+fi
+":"""
+
+"""The executable to be used by the user"""
+import sys, os, re, getpass
+
+myName          = os.path.basename(sys.argv[0])
+myName          = re.sub(".*/", "", myName)
+binDirectory    = os.path.realpath(sys.argv[0])
+rootDirectory   = re.sub("/bin/.*", "", binDirectory)
+libDirectory    = rootDirectory
+
+sys.path.append(libDirectory)
+
+from hodlib.RingMaster.ringMaster import main
+from hodlib.Common.setup import *
+from hodlib.Common.descGenerator import *
+from hodlib.Common.util import local_fqdn, filter_warnings
+from hodlib.Common.logger import hodLog
+from hodlib.Common.util import get_exception_string, get_exception_error_string
+
+filter_warnings()
+
+reVersion = re.compile(".*(\d+_\d+).*")
+
+VERSION = '$HeadURL$'
+
+reMatch = reVersion.match(VERSION)
+if reMatch:
+    VERSION = reMatch.group(1)
+    VERSION = re.sub("_", ".", VERSION)
+else:
+    VERSION = 'DEV'
+
+# Definition tuple is of the form:
+#  (name, type, description, default value, required?, validate?)
+#
+defList = { 'ringmaster' : (
+             ('work-dirs', 'list', 'hod work directories',
+              False, None, True, False),
+
+             ('temp-dir', 'directory', 'Ringmaster temporary directory.',
+              False, None, True, False),
+              
+             ('log-dir', 'directory', 'hod logging directory.', 
+              False, os.path.join(rootDirectory, 'logs'), False, True),
+              
+             ('syslog-address', 'address', 'Syslog address.',
+              False, None, False, True),
+               
+             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
+              False, None, True, True),
+              
+             ('http-port-range', 'range', 'HTTP port range n-m.',
+              False, None, True, True),
+              
+             ('debug', 'pos_int', 'Debugging level, 0-4.',
+              False, 3, True,   True),
+               
+             ('register', 'bool', 'Register with service registry?',
+              False, True, True, True),
+               
+             ('stream', 'bool', 'Output to stderr.',
+              False, False, False, True),
+              
+             ('userid', 'user_account', 
+              'User ID the hod shell is running under.',
+              False, None, True, False),
+               
+             ('svcrgy-addr', 'address', 'Download HTTP address.',
+              False, None, False, True),             
+             
+             ('hadoop-tar-ball', 'string', 'hadoop program tar ball.',
+              False, None, False, False),
+        
+             ('max-connect','pos_int','max connections allowed for a single tarball server',
+             False, 30, False, True),
+
+             ('jt-poll-interval', 'pos_int', 'How often to poll the Job tracker for idleness',
+             False, 120, False, True),
+
+             ('idleness-limit', 'pos_int', 'Limit after which to deallocate the cluster',
+             False, 3600, False, True)),
+
+            'resource_manager' : (
+             ('id', 'string', 'Batch scheduler ID: torque|condor.',
+              False, None, True, True),
+
+             ('pbs-user', 'user_account', 'User ID jobs are submitted under.',
+              False, None, False, True),
+
+             ('pbs-server', 'hostname', 'Hostname of PBS server.',
+              False, None, False, True),    
+
+             ('pbs-account', 'string', 'User Account jobs are submitted under.',
+              False, None, True, False),
+
+             ('queue', 'string', 'Queue of the batch scheduler to query.',
+              False, None, False, False),
+              
+             ('batch-home', 'directory', 'Scheduler installation directory.',
+              False, None, True, True),
+             
+             ('options', 'keyval', 'Options to pass to the scheduler.',
+              False, None, False, True),                    
+
+             ('env-vars', 'keyval', 'Environment variables to pass to the submitted jobs.',
+              False, None, False, True)),
+
+            'gridservice-mapred' : (   
+             ('external', 'bool', "Connect to an already running MapRed?",
+              False, False, True, True),
+                          
+             ('host', 'hostname', 'Mapred hostname.', 
+              False, 'localhost', False, True),
+
+             ('info_port', 'pos_int', 'Mapred info port.',
+              False, None, True, True), 
+             
+             ('tracker_port', 'pos_int', 'Mapred job tracker port.',
+              False, None, True, True),                  
+                          
+             ('cmdline-params', 'keyval', 'Hadoop cmdline key/value list.',
+              False, None, False, False),
+
+             ('server-params', 'keyval', 'Hadoop xml key/value list',
+              False, None, False, False),
+
+             ('final-server-params', 'keyval', 'Hadoop final xml params',
+              False, None, False, False),
+              
+             ('envs', 'keyval', 'environment to run this package in',
+              False, None, False, False),
+              
+             ('java-opts', 'keyval', 'java jvm options',
+              False, None, False, False),
+               
+             ('pkgs', 'directory', "directory where the package is installed",
+              False, None, False, True)), 
+               
+               
+            'gridservice-hdfs' : (
+             ('external', 'bool', "Connect to an already running HDFS?",
+              False, False, True, True),
+              
+             ('host', 'hostname', 'HDFS hostname.', 
+              False, 'localhost', True, True),
+             
+             ('fs_port', 'pos_int', 'HDFS port range.',
+              False, None, True, True),
+              
+             ('info_port', 'pos_int', 'HDFS info port.',
+              False, None, True, True), 
+             
+             ('cmdline-params', 'keyval', 'Hadoop cmdline key/value list.',
+              False, None, False, False),
+
+             ('server-params', 'keyval', 'Hadoop xml key/value list',
+              False, None, False, False),
+
+             ('final-server-params', 'keyval', 'Hadoop final xml params',
+              False, None, False, False),
+           
+             ('envs', 'keyval', 'Environment in which to run this package.',
+              False, None, False, False),
+
+             ('java-opts', 'list', 'java jvm options',
+              False, None, False, False),
+            
+             ('pkgs', 'directory', "directory where the package is installed",
+              False, None, False, True)),          
+             
+             
+            'hodring' : (
+             ('temp-dir', 'directory', 'hod work directories',
+              False, None, True, False),
+              
+             ('log-dir', 'directory', 'hod logging directory.', 
+              False, os.path.join(rootDirectory, 'logs'), False, False), 
+
+             ('log-destination-uri', 'string', 
+              'URI to store logs to, local://some_path or '
+              + 'hdfs://host:port/some_path', 
+              False, None, False, True),             
+
+             ('pkgs', 'directory', 'Path to Hadoop to use in case of uploading to HDFS',
+              False, None, False, True),
+              
+             ('syslog-address', 'address', 'Syslog address.',
+              False, None, False, True),
+          
+             ('java-home', 'directory', 'Java home directory.',
+              False, None, True, False),
+              
+             ('debug', 'pos_int', 'Debugging level, 0-4.',
+              False, 3, True, True),
+               
+             ('register', 'bool', 'Register with service registry?',
+              False, True, True, True),
+               
+             ('stream', 'bool', 'Output to stderr.',
+              False, False, False, True),
+
+             ('userid', 'user_account', 
+              'User ID the hod shell is running under.',
+              False, None, True, False),
+
+             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
+              False, None, True, True),
+               
+             ('http-port-range', 'range', 'HTTP port range n-m.',
+              False, None, True, True),
+               
+             ('command', 'string', 'Command for hodring to run.',
+              False, None, False, True),
+              
+             ('service-id', 'string', 'Service ID.',
+              False, None, False, True),
+              
+             ('download-addr', 'address', 'Download HTTP address.',
+              False, None, False, True),
+               
+             ('svcrgy-addr', 'address', 'Download HTTP address.',
+              False, None, False, True),
+    
+             ('ringmaster-xrs-addr', 'address', 'Ringmaster XML-RPC address.',
+              False, None, False, True),
+ 
+             ('tarball-retry-initial-time', 'pos_float','initial retry time for tarball download',
+              False, 1, False, True),
+              
+             ('tarball-retry-interval', 'pos_float','interval to spread retries for tarball download',
+              False, 3, False, True),
+              
+             ('cmd-retry-initial-time', 'pos_float','initial retry time for getting commands',
+              False, 2, False, True),
+             
+             ('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
+              False, 2, False, True)) 
+
+              }   
+
+             
+defOrder = [ 'ringmaster', 'hodring', 'resource_manager', 
+             'gridservice-mapred', 'gridservice-hdfs' ]
+
+def getLogger(cfg):
+  user = cfg['userid']
+  baseLogger = hodLog('ringmaster')
+  log = baseLogger.add_logger('main')
+  
+  if cfg['stream']:
+    baseLogger.add_stream(level=cfg['debug'], addToLoggerNames=('main',))
+
+  if cfg.has_key('log-dir'):
+    serviceId = os.getenv('PBS_JOBID')
+    if serviceId:
+      logDir = os.path.join(cfg['log-dir'], "%s.%s" % (user, serviceId))
+    else:
+      logDir = os.path.join(cfg['log-dir'], user) 
+    if not os.path.exists(logDir):
+      os.mkdir(logDir)
+
+    baseLogger.add_file(logDirectory=logDir, level=cfg['debug'], 
+             addToLoggerNames=('main',))
+
+  if cfg.has_key('syslog-address'):
+    baseLogger.add_syslog(cfg['syslog-address'], 
+      level=cfg['debug'], addToLoggerNames=('main',))
+    
+  return log
+
+def ensureLogDir(logDir):
+  """Verify that the passed in log directory exists, and if it doesn't
+  create it."""
+  if not os.path.exists(logDir):
+    try:
+      old_mask = os.umask(0)
+      os.makedirs(logDir, 01777)
+      os.umask(old_mask)
+    except Exception, e:
+      print >>sys.stderr, "Could not create log directories %s. Exception: %s. Stack Trace: %s" % (logDir, get_exception_error_string(), get_exception_string())
+      raise e
+      
+
+if __name__ == '__main__':
+  confDef = definition()
+  confDef.add_defs(defList, defOrder)
+  ringMasterOptions = options(confDef, "./%s [OPTIONS]" % myName, VERSION)
+  ensureLogDir(ringMasterOptions['ringmaster']['log-dir'])
+  log = getLogger(ringMasterOptions['ringmaster'])
+
+  try:
+    (status, statusMsgs) = ringMasterOptions.verify()
+    if not status:
+      raise Exception("%s" % statusMsgs)
+    ringMasterOptions['ringmaster']['base-dir'] = rootDirectory 
+    main(ringMasterOptions,log)
+    sys.exit(0)
+  except Exception, e:
+    log.error("bin/ringmaster failed to start.%s. Stack trace follows:\n%s" % (get_exception_error_string(),get_exception_string()))

Added: lucene/hadoop/trunk/src/contrib/hod/build.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/build.xml?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/build.xml (added)
+++ lucene/hadoop/trunk/src/contrib/hod/build.xml Fri Jan  4 10:20:17 2008
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+
+<project name="hod" default="compile">
+    <import file="../build-contrib.xml"/>
+    <target name="compile">
+        <mkdir dir="${build.dir}"/>
+        <copy todir="${build.dir}">
+            <fileset dir="${basedir}"/>
+        </copy>
+    </target>
+    <target name="package" depends="compile">
+        <mkdir dir="${dist.dir}/contrib/${name}"/>
+        <copy todir="${dist.dir}/contrib/${name}">
+            <fileset dir="${build.dir}"/>
+        </copy>
+        <chmod dir="${dist.dir}/contrib/${name}/bin" perm="a+x" includes="*"/>
+    </target>
+</project>

Added: lucene/hadoop/trunk/src/contrib/hod/conf/hodrc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/conf/hodrc?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/conf/hodrc (added)
+++ lucene/hadoop/trunk/src/contrib/hod/conf/hodrc Fri Jan  4 10:20:17 2008
@@ -0,0 +1,45 @@
+[hod]
+stream                          = True
+java-home                       = ${JAVA_HOME}
+cluster                         = ${CLUSTER_NAME}
+cluster-factor                  = 1.8
+xrs-port-range                  = 32768-65536
+debug                           = 3
+allocate-wait-time              = 3600
+
+[ringmaster]
+register                        = True
+stream                          = False
+temp-dir                        = /tmp/hod
+http-port-range                 = 8000-9000
+work-dirs                       = /tmp/hod/1,/tmp/hod/2
+xrs-port-range                  = 32768-65536
+debug                           = 3
+
+[hodring]
+stream                          = False
+temp-dir                        = /tmp/hod
+register                        = True
+java-home                       = ${JAVA_HOME}
+http-port-range                 = 8000-9000
+xrs-port-range                  = 32768-65536
+debug                           = 3
+
+[resource_manager]
+queue                           = ${RM_QUEUE}
+batch-home                      = ${RM_HOME}
+id                              = torque
+#env-vars                       = HOD_PYTHON_HOME=/foo/bar/python-2.5.1/bin/python
+
+[gridservice-mapred]
+external                        = False
+pkgs                            = ${HADOOP_HOME}
+tracker_port                    = 8030
+info_port                       = 50080
+
+[gridservice-hdfs]
+external                        = False
+pkgs                            = ${HADOOP_HOME}
+fs_port                         = 8020
+info_port                       = 50070
+

Added: lucene/hadoop/trunk/src/contrib/hod/config.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/config.txt?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/config.txt (added)
+++ lucene/hadoop/trunk/src/contrib/hod/config.txt Fri Jan  4 10:20:17 2008
@@ -0,0 +1,172 @@
+                     HOD Configuration
+                     =================
+
+1. Introduction:
+================
+
+Configuration options for HOD are organized as sections and options 
+within them. They can be specified in two ways: a configuration file 
+in the INI format, and as command line options to the HOD shell, 
+specified in the format --section.option[=value]. If the same option is 
+specified in both places, the value specified on the command line 
+overrides the value in the configuration file.
+
+To get a simple description of all configuration options, you can type
+    hod --verbose-help
+
+This document explains some of the most important or commonly used
+configuration options in some more detail.
+
+2. Sections:
+============
+
+The following are the various sections in the HOD configuration:
+
+    * hod:                  Options for the HOD client
+    * resource_manager:     Options for specifying which resource 
+                            manager to use, and other parameters for 
+                            using that resource manager
+    * ringmaster:           Options for the RingMaster process, 
+    * hodring:              Options for the HodRing processes
+    * gridservice-mapred:   Options for the MapReduce daemons
+    * gridservice-hdfs:     Options for the HDFS daemons.
+
+The following are some of the important options in the HOD 
+configuration:
+
+3. Important / Commonly Used Configuration Options:
+===================================================
+
+3.1. Common configuration options:
+----------------------------------
+
+Certain configuration options are defined in most of the sections of 
+the HOD configuration. Options defined in a section, are used by the
+process for which that section applies. These options have the same
+meaning, but can have different values in each section.
+
+* temp-dir: Temporary directory for usage by the HOD processes. Make 
+            sure that the users who will run hod have rights to create 
+            directories under the directory specified here.
+
+* debug: A numeric value from 1-4. 4 produces the most log information,
+         and 1 the least.
+
+* log-dir: Directory where log files are stored. By default, this is
+           <install-location>/logs/. The restrictions and notes for the
+           temp-dir variable apply here too.
+
+* xrs-port-range: A range of ports, among which an available port shall
+                  be picked for use to run an XML-RPC server.
+
+* http-port-range: A range of ports, among which an available port shall
+                   be picked for use to run an HTTP server.
+
+* java-home: Location of Java to be used by Hadoop.
+
+3.2 hod options:
+----------------
+
+* cluster: A descriptive name given to the cluster. For Torque, this is
+           specified as a 'Node property' for every node in the cluster.
+           HOD uses this value to compute the number of available nodes.
+
+* client-params: A comma-separated list of hadoop config parameters
+                 specified as key-value pairs. These will be used to
+                 generate a hadoop-site.xml on the submit node that 
+                 should be used for running MapReduce jobs.
+
+3.3 resource_manager options:
+-----------------------------
+
+* queue: Name of the queue configured in the resource manager to which
+         jobs are to be submitted.
+
+* batch-home: Install directory to which 'bin' is appended and under 
+              which the executables of the resource manager can be 
+              found. 
+
+* env-vars: This is a comma separated list of key-value pairs, 
+            expressed as key=value, which would be passed to the jobs 
+            launched on the compute nodes. 
+            For example, if the python installation is 
+            in a non-standard location, one can set the environment
+            variable 'HOD_PYTHON_HOME' to the path to the python 
+            executable. The HOD processes launched on the compute nodes
+            can then use this variable.
+
+3.4 ringmaster options:
+-----------------------
+
+* work-dirs: These are a list of comma separated paths that will serve
+             as the root for directories that HOD generates and passes
+             to Hadoop for use to store DFS / MapReduce data. For e.g.
+             this is where DFS data blocks will be stored. Typically,
+             as many paths are specified as there are disks available
+             to ensure all disks are being utilized. The restrictions
+             and notes for the temp-dir variable apply here too.
+
+3.5 gridservice-hdfs options:
+-----------------------------
+
+* external: If false, this indicates that a HDFS cluster must be 
+            bought up by the HOD system, on the nodes which it 
+            allocates via the allocate command. Note that in that case,
+            when the cluster is de-allocated, it will bring down the 
+            HDFS cluster, and all the data will be lost.
+            If true, it will try and connect to an externally configured
+            HDFS system.
+            Typically, because input for jobs are placed into HDFS
+            before jobs are run, and also the output from jobs in HDFS 
+            is required to be persistent, an internal HDFS cluster is 
+            of little value in a production system. However, it allows 
+            for quick testing.
+
+* host: Hostname of the externally configured NameNode, if any
+
+* fs_port: Port to which NameNode RPC server is bound.
+
+* info_port: Port to which the NameNode web UI server is bound.
+
+* pkgs: Installation directory, under which bin/hadoop executable is 
+        located. This can be used to use a pre-installed version of
+        Hadoop on the cluster.
+
+* server-params: A comma-separated list of hadoop config parameters
+                 specified key-value pairs. These will be used to
+                 generate a hadoop-site.xml that will be used by the
+                 NameNode and DataNodes.
+
+* final-server-params: Same as above, except they will be marked final.
+
+
+3.6 gridservice-mapred options:
+-------------------------------
+
+* external: If false, this indicates that a MapReduce cluster must be
+            bought up by the HOD system on the nodes which it allocates
+            via the allocate command.
+            If true, if will try and connect to an externally 
+            configured MapReduce system.
+
+* host: Hostname of the externally configured JobTracker, if any
+
+* tracker_port: Port to which the JobTracker RPC server is bound
+
+* info_port: Port to which the JobTracker web UI server is bound.
+
+* pkgs: Installation directory, under which bin/hadoop executable is 
+        located
+
+* server-params: A comma-separated list of hadoop config parameters
+                 specified key-value pairs. These will be used to
+                 generate a hadoop-site.xml that will be used by the
+                 JobTracker and TaskTrackers
+
+* final-server-params: Same as above, except they will be marked final.
+
+4. Known Issues:
+================
+
+HOD does not currently handle special characters such as space, comma
+and equals in configuration values.

Added: lucene/hadoop/trunk/src/contrib/hod/getting_started.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/getting_started.txt?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/getting_started.txt (added)
+++ lucene/hadoop/trunk/src/contrib/hod/getting_started.txt Fri Jan  4 10:20:17 2008
@@ -0,0 +1,233 @@
+            Getting Started With Hadoop On Demand (HOD)
+            ===========================================
+
+1. Pre-requisites:
+==================
+
+Hardware:
+HOD requires a minimum of 3 nodes configured through a resource manager.
+
+Software:
+The following components are assumed to be installed before using HOD:
+* Torque:
+  (http://www.clusterresources.com/pages/products/torque-resource-manager.php)
+  Currently HOD supports Torque out of the box. We assume that you are
+  familiar with configuring Torque. You can get information about this
+  from the following link: 
+  http://www.clusterresources.com/wiki/doku.php?id=torque:torque_wiki
+* Python (http://www.python.org/)
+  We require version 2.5.1 of Python.
+    
+The following components can be optionally installed for getting better
+functionality from HOD:
+* Twisted Python: This can be used for improving the scalability of HOD
+  (http://twistedmatrix.com/trac/)
+* Hadoop: HOD can automatically distribute Hadoop to all nodes in the 
+  cluster. However, it can also use a pre-installed version of Hadoop,
+  if it is available on all nodes in the cluster.
+  (http://lucene.apache.org/hadoop)
+  HOD currently supports only Hadoop 0.16, which is under development.
+
+NOTE: HOD configuration requires the location of installs of these 
+components to be the same on all nodes in the cluster. It will also 
+make the configuration simpler to have the same location on the submit
+nodes.
+
+2. Resource Manager Configuration Pre-requisites:
+=================================================
+
+For using HOD with Torque:
+* Install Torque components: pbs_server on a head node, pbs_moms on all
+  compute nodes, and PBS client tools on all compute nodes and submit 
+  nodes.
+* Create a queue for submitting jobs on the pbs_server.
+* Specify a name for all nodes in the cluster, by setting a 'node 
+  property' to all the nodes.
+  This can be done by using the 'qmgr' command. For example:
+  qmgr -c "set node node properties=cluster-name"
+* Ensure that jobs can be submitted to the nodes. This can be done by
+  using the 'qsub' command. For example:
+  echo "sleep 30" | qsub -l nodes=3
+* More information about setting up Torque can be found by referring
+  to the documentation under:
+http://www.clusterresources.com/pages/products/torque-resource-manager.php
+
+3. Setting up HOD:
+==================
+
+* HOD is available under the 'contrib' section of Hadoop under the root
+  directory 'hod'.
+* Distribute the files under this directory to all the nodes in the
+  cluster. Note that the location where the files are copied should be
+  the same on all the nodes.
+* On the node from where you want to run hod, edit the file hodrc 
+  which can be found in the <install dir>/conf directory. This file
+  contains the minimal set of values required for running hod.
+* Specify values suitable to your environment for the following 
+  variables defined in the configuration file. Note that some of these
+  variables are defined at more than one place in the file.
+
+  * ${JAVA_HOME}: Location of Java for Hadoop. Hadoop supports Sun JDK
+    1.5.x
+  * ${CLUSTER_NAME}: Name of the cluster which is specified in the 
+    'node property' as mentioned in resource manager configuration.
+  * ${HADOOP_HOME}: Location of Hadoop installation on the compute and
+    submit nodes.
+  * ${RM_QUEUE}: Queue configured for submiting jobs in the resource
+    manager configuration.
+  * ${RM_HOME}: Location of the resource manager installation on the
+    compute and submit nodes.
+
+* The following environment variables *may* need to be set depending on 
+  your environment. These variables must be defined where you run the 
+  HOD client, and also be specified in the HOD configuration file as the 
+  value of the key resource_manager.env-vars. Multiple variables can be
+  specified as a comma separated list of key=value pairs.
+
+  * HOD_PYTHON_HOME: If you install python to a non-default location 
+    of the compute nodes, or submit nodes, then, this variable must be 
+    defined to point to the python executable in the non-standard 
+    location.
+
+
+NOTE: 
+
+You can also review other configuration options in the file and
+modify them to suit your needs. Refer to the file config.txt for 
+information about the HOD configuration.
+
+
+4. Running HOD:
+===============
+
+4.1 Overview:
+-------------
+
+A typical session of HOD will involve atleast three steps: allocate, 
+run hadoop jobs, deallocate.
+
+4.1.1 Operation allocate
+------------------------
+
+The allocate operation is used to allocate a set of nodes and install and
+provision Hadoop on them. It has the following syntax:
+
+  hod -c config_file -t hadoop_tarball_location -o "allocate \
+                                                cluster_dir number_of_nodes"
+
+The hadoop_tarball_location must be a location on a shared file system
+accesible from all nodes in the cluster. Note, the cluster_dir must exist
+before running the command. If the command completes successfully then
+cluster_dir/hadoop-site.xml will be generated and will contain information
+about the allocated cluster's JobTracker and NameNode.
+
+For example, the following command uses a hodrc file in ~/hod-config/hodrc and
+allocates Hadoop (provided by the tarball ~/share/hadoop.tar.gz) on 10 nodes,
+storing the generated Hadoop configuration in a directory named
+~/hadoop-cluster:
+
+  $ hod -c ~/hod-config/hodrc -t ~/share/hadoop.tar.gz -o "allocate \
+                                                        ~/hadoop-cluster 10"
+
+HOD also supports an environment variable called HOD_CONF_DIR. If this is
+defined, HOD will look for a default hodrc file at $HOD_CONF_DIR/hodrc.
+Defining this allows the above command to also be run as follows:
+
+  $ export HOD_CONF_DIR=~/hod-config
+  $ hod -t ~/share/hadoop.tar.gz -o "allocate ~/hadoop-cluster 10" 
+
+4.1.2 Running Hadoop jobs using the allocated cluster
+-----------------------------------------------------
+
+Now, one can run Hadoop jobs using the allocated cluster in the usual manner:
+
+  hadoop --config cluster_dir hadoop_command hadoop_command_args
+
+Continuing our example, the following command will run a wordcount example on
+the allocated cluster:
+
+  $ hadoop --config ~/hadoop-cluster jar \
+       /path/to/hadoop/hadoop-examples.jar wordcount /path/to/input /path/to/output 
+
+4.1.3 Operation deallocate
+--------------------------
+
+The deallocate operation is used to release an allocated cluster. When
+finished with a cluster, deallocate must be run so that the nodes become free
+for others to use. The deallocate operation has the following syntax:
+
+  hod -o "deallocate cluster_dir"
+
+Continuing our example, the following command will deallocate the cluster:
+
+  $ hod -o "deallocate ~/hadoop-cluster" 
+
+4.2 Command Line Options
+------------------------
+
+This section covers the major command line options available via the hod
+command:
+
+--help
+Prints out the help message to see the basic options.
+
+--verbose-help
+All configuration options provided in the hodrc file can be passed on the
+command line, using the syntax --section_name.option_name[=value]. When
+provided this way, the value provided on command line overrides the option
+provided in hodrc. The verbose-help command lists all the available options in
+the hodrc file. This is also a nice way to see the meaning of the
+configuration options.
+
+-c config_file
+Provides the configuration file to use. Can be used with all other options of
+HOD. Alternatively, the HOD_CONF_DIR environment variable can be defined to
+specify a directory that contains a file named hodrc, alleviating the need to
+specify the configuration file in each HOD command.
+
+-b 1|2|3|4
+Enables the given debug level. Can be used with all other options of HOD. 4 is
+most verbose.
+
+-o "help"
+Lists the operations available in the operation mode.
+
+-o "allocate cluster_dir number_of_nodes"
+Allocates a cluster on the given number of cluster nodes, and store the
+allocation information in cluster_dir for use with subsequent hadoop commands.
+Note that the cluster_dir must exist before running the command.
+
+-o "list"
+Lists the clusters allocated by this user. Information provided includes the
+Torque job id corresponding to the cluster, the cluster directory where the
+allocation information is stored, and whether the Map/Reduce daemon is still
+active or not.
+
+-o "info cluster_dir"
+Lists information about the cluster whose allocation information is stored in
+the specified cluster directory.
+
+-o "deallocate cluster_dir"
+Deallocates the cluster whose allocation information is stored in the
+specified cluster directory.
+
+-t hadoop_tarball
+Provisions Hadoop from the given tar.gz file. This option is only applicable
+to the allocate operation. For better distribution performance it is
+recommended that the Hadoop tarball contain only the libraries and binaries,
+and not the source or documentation. 
+
+-Mkey1=value1 -Mkey2=value2
+Provides configuration parameters for the provisioned Map/Reduce daemons
+(JobTracker and TaskTrackers). A hadoop-site.xml is generated with these
+values on the cluster nodes
+
+-Hkey1=value1 -Hkey2=value2
+Provides configuration parameters for the provisioned HDFS daemons (NameNode
+and DataNodes). A hadoop-site.xml is generated with these values on the
+cluster nodes
+
+-Ckey1=value1 -Ckey2=value2
+Provides configuration parameters for the client from where jobs can be
+submitted. A hadoop-site.xml is generated with these values on the submit
+node.

Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/__init__.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/__init__.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/__init__.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/__init__.py Fri Jan  4 10:20:17 2008
@@ -0,0 +1,16 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/__init__.py
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/__init__.py
------------------------------------------------------------------------------
    svn:executable = *

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/__init__.py
------------------------------------------------------------------------------
    svn:keywords = Id Revision HeadURL

Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/goldAllocationManager.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/goldAllocationManager.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/goldAllocationManager.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/goldAllocationManager.py Fri Jan  4 10:20:17 2008
@@ -0,0 +1,104 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+"""Gold Allocation Manager Implementation"""
+# -*- python -*-
+
+import sys, httplib
+import sha, base64, hmac
+import xml.dom.minidom
+
+from hodlib.Common.util import *
+
+class goldAllocationManager:
+  def __init__(self, cfg, log):
+    self.__GOLD_SECRET_KEY_FILE = cfg['auth-file']
+    (self.__goldHost, self.__goldPort) = (cfg['allocation-manager-address'][0], 
+                                          cfg['allocation-manager-address'][1])
+    self.cfg = cfg
+    self.log = log
+
+  def getQuote(self, user, project, ignoreErrors=True):
+    # Get Secret Key from File
+    secret = ''
+    try:
+      secretFile = open(self.__GOLD_SECRET_KEY_FILE)
+      secret = secretFile.readline()
+    except Exception, e:
+      self.log.error("Unable to open file %s" % self.__GOLD_SECRET_KEY_FILE)
+      self.log.debug(get_exception_string())
+      return (ignoreErrors or False)
+    secretFile.close()
+    secret = secret.rstrip()
+
+    # construct the SSRMAP request body 
+    body = '<Body><Request action="Quote" actor="hod"><Object>Job</Object><Data><Job><ProjectId>%s</ProjectId><UserId>%s</UserId><WallDuration>10</WallDuration></Job></Data></Request></Body>' % (project, user)
+
+    # compute digest
+    message = sha.new()
+    message.update(body)
+    digest = message.digest()
+    digestStr = base64.b64encode(digest)
+
+    # compute signature
+    message = hmac.new(secret, digest, sha)
+    signatureStr = base64.b64encode(message.digest())
+
+    # construct the SSSRMAP Message
+    sssrmapRequest = '<?xml version="1.0" encoding="UTF-8"?>\
+<Envelope>%s<Signature><DigestValue>%s</DigestValue><SignatureValue>%s</SignatureValue><SecurityToken type="Symmetric"></SecurityToken></Signature></Envelope>' % (body, digestStr, signatureStr)
+    self.log.info('sssrmapRequest: %s' % sssrmapRequest)
+
+    try:
+      # post message to GOLD server
+      webservice = httplib.HTTP(self.__goldHost, self.__goldPort)
+      webservice.putrequest("POST", "/SSSRMAP3 HTTP/1.1")
+      webservice.putheader("Content-Type", "text/xml; charset=\"utf-8\"")
+      webservice.putheader("Transfer-Encoding", "chunked")
+      webservice.endheaders()
+      webservice.send("%X" % len(sssrmapRequest) + "\r\n" + sssrmapRequest + '0\r\n')
+
+      # handle the response
+      statusCode, statusmessage, header = webservice.getreply()
+      responseStr = webservice.getfile().read()
+      self.log.debug("httpStatusCode: %d" % statusCode)
+      self.log.info('responseStr: %s' % responseStr)
+
+      # parse XML response
+      if (statusCode == 200):
+        responseArr = responseStr.split("\n")
+        responseBody = responseArr[2]
+        try:
+          doc = xml.dom.minidom.parseString(responseBody)
+          responseVal = doc.getElementsByTagName("Value")[0].firstChild.nodeValue
+          self.log.info("responseVal: %s" % responseVal)
+          if (responseVal == 'Success'):
+            return True
+          else:
+            return False
+        except Exception, e:
+          self.log.error("Unable to parse GOLD responseBody XML \"(%s)\" to get responseVal" % (responseBody))
+          self.log.debug(get_exception_string())
+          return (ignoreErrors or False)
+      else:
+        self.log.error("Invalid HTTP statusCode %d" % statusCode)
+    except Exception, e:
+      self.log.error("Unable to POST message to GOLD server (%s, %d)" %
+                       (self.__goldHost, self.__goldPort))
+      self.log.debug(get_exception_string())
+      return (ignoreErrors or False)
+
+    return True
+

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/goldAllocationManager.py
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/goldAllocationManager.py
------------------------------------------------------------------------------
    svn:executable = *

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/AllocationManagers/goldAllocationManager.py
------------------------------------------------------------------------------
    svn:keywords = Id Revision HeadURL

Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/__init__.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/__init__.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/__init__.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/__init__.py Fri Jan  4 10:20:17 2008
@@ -0,0 +1,15 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/__init__.py
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/__init__.py
------------------------------------------------------------------------------
    svn:executable = *

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/__init__.py
------------------------------------------------------------------------------
    svn:keywords = Id Revision HeadURL

Added: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/allocationManagerUtil.py
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/allocationManagerUtil.py?rev=608950&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/allocationManagerUtil.py (added)
+++ lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/allocationManagerUtil.py Fri Jan  4 10:20:17 2008
@@ -0,0 +1,27 @@
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+"""defines Allocation Manager Utilities"""
+
+# -*- python -*-
+from hodlib.allocationManagers.goldAllocationManager import goldAllocationManager
+
+class allocationManagerUtil:
+  def getAllocationManager(name, cfg, log):
+    """returns a concrete instance of the specified AllocationManager"""
+    if name == 'gold':
+      return goldAllocationManager(cfg, log)
+    
+  getAllocationManager = staticmethod(getAllocationManager)

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/allocationManagerUtil.py
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/allocationManagerUtil.py
------------------------------------------------------------------------------
    svn:executable = *

Propchange: lucene/hadoop/trunk/src/contrib/hod/hodlib/Common/allocationManagerUtil.py
------------------------------------------------------------------------------
    svn:keywords = Id Revision HeadURL



Mime
View raw message