hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yhema...@apache.org
Subject svn commit: r642139 - in /hadoop/core/trunk/src/contrib/hod: hodlib/GridServices/hdfs.py hodlib/GridServices/mapred.py hodlib/Hod/hadoop.py testing/testHadoop.py
Date Fri, 28 Mar 2008 08:40:23 GMT
Author: yhemanth
Date: Fri Mar 28 01:40:08 2008
New Revision: 642139

URL: http://svn.apache.org/viewvc?rev=642139&view=rev
Log:
HADOOP-3103. Sets hadoop.tmp.dir to path under hod.temp-dir instead of cluster directory.
Contributed by Vinod Kumar Vavilapalli.

Modified:
    hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py
    hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py
    hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py
    hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py

Modified: hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py?rev=642139&r1=642138&r2=642139&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py (original)
+++ hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/hdfs.py Fri Mar 28 01:40:08 2008
@@ -149,6 +149,7 @@
   
   def _setWorkDirs(self, workDirs, envs, attrs, parentDirs, subDir):
     namedir = None
+    hadooptmpdir = None
     datadir = []
 
     for p in parentDirs:
@@ -156,6 +157,9 @@
       workDirs.append(os.path.join(p, subDir))
       dir = os.path.join(p, subDir, 'dfs-data')
       datadir.append(dir)
+      if not hadooptmpdir:
+        # Not used currently, generating hadooptmpdir just in case
+        hadooptmpdir = os.path.join(p, subDir, 'hadoop-tmp')
 
       if not namedir:
         namedir = os.path.join(p, subDir, 'dfs-name')
@@ -165,6 +169,7 @@
 
     # FIXME!! use csv
     attrs['dfs.name.dir'] = namedir
+    attrs['hadoop.tmp.dir'] = hadooptmpdir
     attrs['dfs.data.dir'] = ','.join(datadir)
     # FIXME -- change dfs.client.buffer.dir
     envs['HADOOP_ROOT_LOGGER'] = "INFO,DRFA"

Modified: hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py?rev=642139&r1=642138&r2=642139&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py (original)
+++ hadoop/core/trunk/src/contrib/hod/hodlib/GridServices/mapred.py Fri Mar 28 01:40:08 2008
@@ -153,6 +153,7 @@
     local = []
     system = None
     temp = None
+    hadooptmpdir = None
     dfsclient = []
     
     for p in parentDirs:
@@ -164,6 +165,9 @@
         system = os.path.join(p, subDir, 'mapred-system')
       if not temp:
         temp = os.path.join(p, subDir, 'mapred-temp')
+      if not hadooptmpdir:
+        # Not used currently, generating hadooptmpdir just in case
+        hadooptmpdir = os.path.join(p, subDir, 'hadoop-tmp')
       dfsclientdir = os.path.join(p, subDir, 'dfs-client')
       dfsclient.append(dfsclientdir)
       workDirs.append(dfsclientdir)
@@ -172,6 +176,7 @@
     attrs['mapred.system.dir'] = 'fillindir'
     attrs['mapred.temp.dir'] = temp
     attrs['dfs.client.buffer.dir'] = ','.join(dfsclient)
+    attrs['hadoop.tmp.dir'] = hadooptmpdir
 
 
     envs['HADOOP_ROOT_LOGGER'] = "INFO,DRFA"

Modified: hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py?rev=642139&r1=642138&r2=642139&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py (original)
+++ hadoop/core/trunk/src/contrib/hod/hodlib/Hod/hadoop.py Fri Mar 28 01:40:08 2008
@@ -73,8 +73,10 @@
     description = {}
     paramsDict = {  'mapred.job.tracker'    : mapredAddr , \
                     'fs.default.name'       : "hdfs://" + hdfsAddr, \
-                    'hadoop.tmp.dir'        : confDir, \
-                    'dfs.client.buffer.dir' : tempDir, }
+                    'hadoop.tmp.dir'        : tempDir, \
+                    'dfs.client.buffer.dir' : os.path.join(tempDir, 'dfs',
+                                                                    'tmp'),
+                 }
 
     paramsDict['mapred.system.dir'] = mrSysDir
     

Modified: hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py?rev=642139&r1=642138&r2=642139&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py (original)
+++ hadoop/core/trunk/src/contrib/hod/testing/testHadoop.py Fri Mar 28 01:40:08 2008
@@ -92,6 +92,9 @@
 
     # fs.default.name should start with hdfs://
     assert(keyvals['fs.default.name'].startswith('hdfs://'))
+    assert(keyvals['hadoop.tmp.dir'] == self.tempDir)
+    assert(keyvals['dfs.client.buffer.dir'] == os.path.join(self.tempDir,
+                                                            'dfs', 'tmp'))
 
     # TODO other tests
     pass



Mime
View raw message