hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1056163 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobTracker.java src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java
Date Fri, 07 Jan 2011 02:13:25 GMT
Author: todd
Date: Fri Jan  7 02:13:25 2011
New Revision: 1056163

URL: http://svn.apache.org/viewvc?rev=1056163&view=rev
Log:
JobTracker should not try to remove mapred.system.dir during startup. Contributed by Todd
Lipcon

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1056163&r1=1056162&r2=1056163&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Jan  7 02:13:25 2011
@@ -463,6 +463,9 @@ Release 0.22.0 - Unreleased
     MAPREDUCE-2234. If Localizer can't create task log directory, it should fail
     on the spot. (todd)
 
+    MAPREDUCE-2219. JobTracker should not try to remove mapred.system.dir
+    during startup. (todd)
+
 Release 0.21.1 - Unreleased
 
   NEW FEATURES

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1056163&r1=1056162&r2=1056163&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Jan  7 02:13:25
2011
@@ -1583,22 +1583,33 @@ public class JobTracker implements MRCon
             break; // if there is something to recover else clean the sys dir
           }
         }
-        LOG.info("Cleaning up the system directory");
-        fs.delete(systemDir, true);
-        if (FileSystem.mkdirs(fs, systemDir, 
-            new FsPermission(SYSTEM_DIR_PERMISSION))) {
+        if (!fs.exists(systemDir)) {
+          LOG.info("Creating the system directory");
+          if (FileSystem.mkdirs(fs, systemDir, 
+                                new FsPermission(SYSTEM_DIR_PERMISSION))) {
+            // success
+            break;
+          } else {
+            LOG.error("Mkdirs failed to create " + systemDir);
+          }
+        } else {
+          // It exists, just set permissions and clean the contents up
+          LOG.info("Cleaning up the system directory");
+          fs.setPermission(systemDir, new FsPermission(SYSTEM_DIR_PERMISSION));
+          deleteContents(fs, systemDir);
           break;
         }
-        LOG.error("Mkdirs failed to create " + systemDir);
       } catch (AccessControlException ace) {
-        LOG.warn("Failed to operate on " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir 
+        LOG.warn("Failed to operate on " + JTConfig.JT_SYSTEM_DIR + "("
+                 + systemDir.makeQualified(fs)
                  + ") because of permissions.");
-        LOG.warn("Manually delete the " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir 
-                 + ") and then start the JobTracker.");
+        LOG.warn("This directory should exist and be owned by the user '" +
+                 UserGroupInformation.getCurrentUser() + "'");
         LOG.warn("Bailing out ... ");
         throw ace;
       } catch (IOException ie) {
-        LOG.info("problem cleaning system directory: " + systemDir, ie);
+        LOG.info("problem cleaning system directory: " +
+                 systemDir.makeQualified(fs), ie);
       }
       Thread.sleep(FS_ACCESS_RETRY_PERIOD);
     }
@@ -1634,6 +1645,18 @@ public class JobTracker implements MRCon
     completedJobStatusStore = new CompletedJobStatusStore(conf, aclsManager);
   }
 
+  /**
+   * Recursively delete the contents of a directory without deleting the
+   * directory itself.
+   */
+  private void deleteContents(FileSystem fs, Path dir) throws IOException {
+    for (FileStatus stat : fs.listStatus(dir)) {
+      if (!fs.delete(stat.getPath(), true)) {
+        throw new IOException("Unable to delete " + stat.getPath());
+      }
+    }
+  }
+
   private static SimpleDateFormat getDateFormat() {
     return new SimpleDateFormat("yyyyMMddHHmm");
   }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java?rev=1056163&r1=1056162&r2=1056163&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestMapredSystemDir.java
Fri Jan  7 02:13:25 2011
@@ -44,8 +44,10 @@ public class TestMapredSystemDir extends
   // mapred ugi
   private static final UserGroupInformation MR_UGI = 
     TestMiniMRWithDFSWithDistinctUsers.createUGI("mr", false);
+  private static final FsPermission SYSTEM_DIR_PARENT_PERMISSION =
+    FsPermission.createImmutable((short) 0755); // rwxr-xr-x
   private static final FsPermission SYSTEM_DIR_PERMISSION =
-    FsPermission.createImmutable((short) 0733); // rwx-wx-wx
+    FsPermission.createImmutable((short) 0700); // rwx------
   
   public void testGarbledMapredSystemDir() throws Exception {
     Configuration conf = new Configuration();
@@ -60,17 +62,21 @@ public class TestMapredSystemDir extends
         }
       });
       
-      // create Configs.SYSTEM_DIR's parent (the parent has to be given 
-      // permissions since the JT internally tries to delete the leaf of
-      // the directory structure
-      Path mapredSysDir = 
-        new Path(conf.get(JTConfig.JT_SYSTEM_DIR)).getParent();
+
+      // create Configs.SYSTEM_DIR's parent with restrictive permissions.
+      // So long as the JT has access to the system dir itself it should
+      // be able to start.
+      Path mapredSysDir = new Path(conf.get(JTConfig.JT_SYSTEM_DIR));
+      Path parentDir =  mapredSysDir.getParent();
+      fs.mkdirs(parentDir);
+      fs.setPermission(parentDir,
+                       new FsPermission(SYSTEM_DIR_PARENT_PERMISSION));
       fs.mkdirs(mapredSysDir);
       fs.setPermission(mapredSysDir, new FsPermission(SYSTEM_DIR_PERMISSION));
       fs.setOwner(mapredSysDir, "mr", "mrgroup");
 
       // start mr (i.e jobtracker)
-      Configuration mrConf = new Configuration();
+      Configuration mrConf = new Configuration(conf);
       mr = new MiniMRCluster(0, 0, 0, dfs.getFileSystem().getUri().toString(),
                              1, null, null, MR_UGI, new JobConf(mrConf));
       JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
@@ -105,4 +111,4 @@ public class TestMapredSystemDir extends
       if (mr != null) { mr.shutdown();}
     }
   }
-}
\ No newline at end of file
+}



Mime
View raw message