hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077689 - in /hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred: JvmManager.java TaskTracker.java
Date Fri, 04 Mar 2011 04:44:47 GMT
Author: omalley
Date: Fri Mar  4 04:44:46 2011
New Revision: 1077689

URL: http://svn.apache.org/viewvc?rev=1077689&view=rev
Log:
commit da204a8d0d5d96ed14f0390cf7a0d7db198201ae
Author: Devaraj Das <ddas@yahoo-inc.com>
Date:   Fri Sep 17 14:59:30 2010 -0700

    Makes the TT check the task logs size after the jvm exits, and if the task logs size is
more than the configured limit, the TT wipes it out

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java?rev=1077689&r1=1077688&r2=1077689&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JvmManager.java
Fri Mar  4 04:44:46 2011
@@ -30,6 +30,9 @@ import java.util.Vector;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.CleanupQueue.PathDeletionContext;
 import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.server.tasktracker.JVMInfo;
@@ -177,6 +180,24 @@ class JvmManager {
                                         workDir.substring(userDir.length())));
                                            
   }
+  
+  static void checkAndDeleteTaskLogs(TaskTracker tracker, Task firstTask) {
+    File logdir = TaskLog.getAttemptDir(firstTask.getTaskID(), 
+        firstTask.isTaskCleanupTask());
+    //allow for 10% over the limit
+    if (FileUtil.getDU(logdir) >
+        (1.1 * tracker.getRetainSize(firstTask.getTaskID()))){
+      LOG.info("Deleting user log path since the amount of data in the logs" +
+      		" exceeded the allowed " +
+      		"log limits " + logdir);
+      String user = firstTask.getUser();
+      String jobLogDir = firstTask.getJobID().toString() + logdir.getName();
+      PathDeletionContext item = 
+        new TaskController.DeletionContext(tracker.getTaskController(),
+            true, user, jobLogDir);
+      tracker.getCleanupThread().addToQueue(item);
+    }
+  }
 
   static class JvmManagerForType {
     //Mapping from the JVM IDs to running Tasks
@@ -479,6 +500,7 @@ class JvmManager {
           LOG.info("JVM : " + jvmId + " exited with exit code " + exitCode
               + ". Number of tasks it ran: " + numTasksRan);
           deleteWorkDir(tracker, firstTask);
+          checkAndDeleteTaskLogs(tracker, firstTask);
         }
       }
 

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1077689&r1=1077688&r2=1077689&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Fri Mar  4 04:44:46 2011
@@ -259,6 +259,8 @@ public class TaskTracker implements MRCo
   private int maxMapSlots;
   private int maxReduceSlots;
   private int failures;
+  final long mapRetainSize;
+  final long reduceRetainSize;
 
   private ACLsManager aclsManager;
   
@@ -1204,6 +1206,8 @@ public class TaskTracker implements MRCo
   TaskTracker() {
     server = null;
     workerThreads = 0;
+    mapRetainSize = TaskLogsTruncater.DEFAULT_RETAIN_SIZE;
+    reduceRetainSize = TaskLogsTruncater.DEFAULT_RETAIN_SIZE;
   }
 
   void setConf(JobConf conf) {
@@ -1262,6 +1266,10 @@ public class TaskTracker implements MRCo
     server.start();
     this.httpPort = server.getPort();
     checkJettyPort(httpPort);
+    mapRetainSize = conf.getLong(TaskLogsTruncater.MAP_USERLOG_RETAIN_SIZE, 
+        TaskLogsTruncater.DEFAULT_RETAIN_SIZE);
+    reduceRetainSize = conf.getLong(TaskLogsTruncater.REDUCE_USERLOG_RETAIN_SIZE,
+        TaskLogsTruncater.DEFAULT_RETAIN_SIZE);
   }
 
   private void checkJettyPort(int port) throws IOException { 
@@ -1633,6 +1641,10 @@ public class TaskTracker implements MRCo
     return totalMemoryAllottedForTasks;
   }
 
+  long getRetainSize(org.apache.hadoop.mapreduce.TaskAttemptID tid) {
+    return tid.isMap() ? mapRetainSize : reduceRetainSize;
+  }
+  
   /**
    * Check if the jobtracker directed a 'reset' of the tasktracker.
    * 



Mime
View raw message