hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077712 - in /hadoop/common/branches/branch-0.20-security-patches/src: c++/task-controller/impl/ mapred/org/apache/hadoop/mapred/
Date Fri, 04 Mar 2011 04:47:11 GMT
Author: omalley
Date: Fri Mar  4 04:47:11 2011
New Revision: 1077712

URL: http://svn.apache.org/viewvc?rev=1077712&view=rev
Log:
commit 0914a7c372060c897c0cb2c834f94b1c4c3694c6
Author: Chris Douglas <cdouglas@apache.org>
Date:   Mon Sep 20 00:47:52 2010 -0700

    Add LTC debugging, singleton CleanupQueue pattern, remove tabs in task-controller.c

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/c++/task-controller/impl/task-controller.c
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/UserLogCleaner.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/c++/task-controller/impl/task-controller.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/c%2B%2B/task-controller/impl/task-controller.c?rev=1077712&r1=1077711&r2=1077712&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/c++/task-controller/impl/task-controller.c
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/c++/task-controller/impl/task-controller.c
Fri Mar  4 04:47:11 2011
@@ -903,8 +903,8 @@ static int delete_path(const char *full_
     // check to make sure the directory exists
     if (access(full_path, F_OK) != 0) {
       if (errno == ENOENT) {
-	free(paths[0]);
-	return 0;
+        free(paths[0]);
+        return 0;
       }
     }
     FTS* tree = fts_open(paths, FTS_PHYSICAL | FTS_XDEV, NULL);
@@ -922,14 +922,14 @@ static int delete_path(const char *full_
       switch (entry->fts_info) {
 
       case FTS_DP:        // A directory being visited in post-order
-	if (!needs_tt_user ||
+        if (!needs_tt_user ||
             strcmp(entry->fts_path, full_path) != 0) {
-	  if (rmdir(entry->fts_accpath) != 0) {
-	    fprintf(LOGFILE, "Couldn't delete directory %s - %s\n", 
-		    entry->fts_path, strerror(errno));
-	    exit_code = -1;
-	  }
-	}
+          if (rmdir(entry->fts_accpath) != 0) {
+            fprintf(LOGFILE, "Couldn't delete directory %s - %s\n", 
+                    entry->fts_path, strerror(errno));
+            exit_code = -1;
+          }
+        }
         break;
 
       case FTS_F:         // A regular file
@@ -939,7 +939,7 @@ static int delete_path(const char *full_
         if (unlink(entry->fts_accpath) != 0) {
           fprintf(LOGFILE, "Couldn't delete file %s - %s\n", entry->fts_path,
                   strerror(errno));
-	  exit_code = -1;
+          exit_code = -1;
         }
         break;
 
@@ -957,13 +957,13 @@ static int delete_path(const char *full_
             fprintf(LOGFILE, "Error chmoding %s - %s, continuing\n", 
                     entry->fts_path, strerror(errno));
           }
-	}
+        }
         break;
 
       case FTS_NS:        // A file with no stat(2) information
         // usually a root directory that doesn't exist
         fprintf(LOGFILE, "Directory not found %s\n", entry->fts_path);
-	break;
+        break;
 
       case FTS_DC:        // A directory that causes a cycle
       case FTS_DOT:       // A dot directory

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java?rev=1077712&r1=1077711&r2=1077712&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/CleanupQueue.java
Fri Mar  4 04:47:11 2011
@@ -32,7 +32,11 @@ class CleanupQueue {
   public static final Log LOG =
     LogFactory.getLog(CleanupQueue.class);
 
-  private static PathCleanupThread cleanupThread;
+  private static final PathCleanupThread cleanupThread =
+    new PathCleanupThread();
+  private static final CleanupQueue inst = new CleanupQueue();
+
+  public static CleanupQueue getInstance() { return inst; }
 
   /**
    * Create a singleton path-clean-up queue. It can be used to delete
@@ -42,20 +46,14 @@ class CleanupQueue {
    * {@link CleanupQueue#addToQueue(PathDeletionContext...)} to add paths for
    * deletion.
    */
-  public CleanupQueue() {
-    synchronized (PathCleanupThread.class) {
-      if (cleanupThread == null) {
-        cleanupThread = new PathCleanupThread();
-      }
-    }
-  }
+  private CleanupQueue() { }
   
   /**
    * Contains info related to the path of the file/dir to be deleted
    */
   static class PathDeletionContext {
-    Path fullPath;// full path of file or dir
-    Configuration conf;
+    final Path fullPath;// full path of file or dir
+    final Configuration conf;
 
     public PathDeletionContext(Path fullPath, Configuration conf) {
       this.fullPath = fullPath;
@@ -70,14 +68,21 @@ class CleanupQueue {
      * Deletes the path (and its subdirectories recursively)
      */
     protected void deletePath() throws IOException {
-      fullPath.getFileSystem(conf).delete(fullPath, true);
+      final Path p = getPathForCleanup();
+      p.getFileSystem(conf).delete(p, true);
+    }
+
+    @Override
+    public String toString() {
+      final Path p = getPathForCleanup();
+      return (null == p) ? "undefined" : p.toString();
     }
   }
 
   /**
    * Adds the paths to the queue of paths to be deleted by cleanupThread.
    */
-  void addToQueue(PathDeletionContext... contexts) {
+  public void addToQueue(PathDeletionContext... contexts) {
     cleanupThread.addToQueue(contexts);
   }
 
@@ -102,6 +107,7 @@ class CleanupQueue {
       for (PathDeletionContext context : contexts) {
         try {
           queue.put(context);
+          LOG.debug("Added " + context + " to queue");
         } catch(InterruptedException ie) {}
       }
     }
@@ -117,13 +123,13 @@ class CleanupQueue {
           context.deletePath();
           // delete the path.
           if (LOG.isDebugEnabled()) {
-            LOG.debug("DELETED " + context.fullPath);
+            LOG.debug("DELETED " + context);
           }
         } catch (InterruptedException t) {
-          LOG.warn("Interrupted deletion of " + context.fullPath);
+          LOG.warn("Interrupted deletion of " + context);
           return;
-        } catch (Exception e) {
-          LOG.warn("Error deleting path " + context.fullPath + ": " + e);
+        } catch (Throwable e) {
+          LOG.warn("Error deleting path " + context, e);
         } 
       }
     }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1077712&r1=1077711&r2=1077712&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Mar  4 04:47:11 2011
@@ -3140,7 +3140,8 @@ public class JobInProgress {
         }
 
         Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID());
-        new CleanupQueue().addToQueue(new PathDeletionContext(tempDir, conf)); 
+        CleanupQueue.getInstance().addToQueue(
+            new PathDeletionContext(tempDir, conf)); 
       } catch (IOException e) {
         LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
       }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java?rev=1077712&r1=1077711&r2=1077712&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/LinuxTaskController.java
Fri Mar  4 04:47:11 2011
@@ -281,6 +281,9 @@ class LinuxTaskController extends TaskCo
                    Integer.toString(Commands.DELETE_AS_USER.getValue()),
                    subDir};
     ShellCommandExecutor shExec = new ShellCommandExecutor(command);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("deleteAsUser: " + Arrays.toString(command));
+    }
     shExec.execute();
   }
 
@@ -292,6 +295,9 @@ class LinuxTaskController extends TaskCo
                    Integer.toString(Commands.DELETE_LOG_AS_USER.getValue()),
                    subDir};
     ShellCommandExecutor shExec = new ShellCommandExecutor(command);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("deleteLogAsUser: " + Arrays.toString(command));
+    }
     shExec.execute();
   }
 
@@ -305,6 +311,9 @@ class LinuxTaskController extends TaskCo
                    Integer.toString(taskPid),
                    Integer.toString(signal.getValue())};
     ShellCommandExecutor shExec = new ShellCommandExecutor(command);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("signalTask: " + Arrays.toString(command));
+    }
     try {
       shExec.execute();
     } catch (ExitCodeException e) {

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java?rev=1077712&r1=1077711&r2=1077712&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskController.java
Fri Mar  4 04:47:11 2011
@@ -171,6 +171,12 @@ public abstract class TaskController imp
         controller.deleteAsUser(user, subDir);
       }
     }
+
+    @Override
+    public String toString() {
+      return (isLog ? "log(" : "dir(") +
+        user + "," + subDir + ")";
+    }
   }
   
    /**

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1077712&r1=1077711&r2=1077712&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Fri Mar  4 04:47:11 2011
@@ -1285,7 +1285,7 @@ public class TaskTracker implements MRCo
   private void startCleanupThreads() throws IOException {
     taskCleanupThread.setDaemon(true);
     taskCleanupThread.start();
-    directoryCleanupThread = new CleanupQueue();
+    directoryCleanupThread = CleanupQueue.getInstance();
   }
 
   // only used by tests

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/UserLogCleaner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/UserLogCleaner.java?rev=1077712&r1=1077711&r2=1077712&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/UserLogCleaner.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/UserLogCleaner.java
Fri Mar  4 04:47:11 2011
@@ -61,7 +61,7 @@ public class UserLogCleaner extends Thre
     this.userLogManager = userLogManager;
     threadSleepTime = conf.getLong(USERLOGCLEANUP_SLEEPTIME,
         DEFAULT_THREAD_SLEEP_TIME);
-    cleanupQueue = new CleanupQueue();
+    cleanupQueue = CleanupQueue.getInstance();
     localFs = FileSystem.getLocal(conf);
     setClock(new Clock());
     setDaemon(true);



Mime
View raw message