hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r747547 - in /hadoop/core/trunk: CHANGES.txt src/contrib/chukwa/conf/system-data-loader.properties src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
Date Tue, 24 Feb 2009 21:12:33 GMT
Author: cdouglas
Date: Tue Feb 24 21:12:33 2009
New Revision: 747547

URL: http://svn.apache.org/viewvc?rev=747547&view=rev
Log:
HADOOP-5042. Add a cleanup stage to log rollover in Chukwa appender. Contributed by Jerome
Boulon.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=747547&r1=747546&r2=747547&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Feb 24 21:12:33 2009
@@ -131,6 +131,9 @@
 
     HADOOP-4191. Adds a testcase for JobHistory. (Ravi Gummadi via ddas)
 
+    HADOOP-5042. Add a cleanup stage to log rollover in Chukwa appender.
+    (Jerome Boulon via cdouglas)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties?rev=747547&r1=747546&r2=747547&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties (original)
+++ hadoop/core/trunk/src/contrib/chukwa/conf/system-data-loader.properties Tue Feb 24 21:12:33
2009
@@ -2,6 +2,8 @@
 log4j.appender.R=org.apache.hadoop.chukwa.inputtools.log4j.ChukwaDailyRollingFileAppender
 log4j.appender.R.File=${CHUKWA_LOG_DIR}/${RECORD_TYPE}.log
 log4j.appender.R.recordType=${RECORD_TYPE}
+log4j.appender.R.maxBackupIndex=3
+log4j.appender.R.cleanUpRegex=$fileName.20*
 log4j.appender.R.layout=org.apache.log4j.PatternLayout
 log4j.appender.R.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
 

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java?rev=747547&r1=747546&r2=747547&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/inputtools/log4j/ChukwaDailyRollingFileAppender.java
Tue Feb 24 21:12:33 2009
@@ -10,13 +10,18 @@
 package org.apache.hadoop.chukwa.inputtools.log4j;
 
 import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Collections;
 import java.util.Date;
 import java.util.GregorianCalendar;
+import java.util.List;
 import java.util.Locale;
 import java.util.TimeZone;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
 import org.apache.hadoop.chukwa.util.RecordConstants;
@@ -130,8 +135,7 @@
     <b>DatePattern</b> option. The text before the colon is interpeted
     as the protocol specificaion of a URL which is probably not what
     you want. */
-
-
+    
 public class ChukwaDailyRollingFileAppender extends FileAppender {
 
 	static Logger log = Logger.getLogger(ChukwaDailyRollingFileAppender.class);
@@ -172,6 +176,16 @@
     The next time we estimate a rollover should occur. */
   private long nextCheck = System.currentTimeMillis () - 1;
 
+  /**
+   * Regex to select log files to be deleted
+   */
+  private String cleanUpRegex = null;
+  
+  /**
+   * Set the maximum number of backup files to keep around.
+   */
+  private int maxBackupIndex = 10;
+  
   Date now = new Date();
 
   SimpleDateFormat sdf;
@@ -351,6 +365,7 @@
     }
 
     File file = new File(fileName);
+    
     boolean result = file.renameTo(target);
     if(result) {
       LogLog.debug(fileName +" -> "+ scheduledFilename);
@@ -367,8 +382,82 @@
       errorHandler.error("setFile("+fileName+", false) call failed.");
     }    
     scheduledFilename = datedFilename;
+    cleanUp();
+  }
+
+  
+  public String getCleanUpRegex() {
+    return cleanUpRegex;
+  }
+
+  public void setCleanUpRegex(String cleanUpRegex) {
+    this.cleanUpRegex = cleanUpRegex;
+  }
+
+  public int getMaxBackupIndex() {
+    return maxBackupIndex;
+  }
+
+  public void setMaxBackupIndex(int maxBackupIndex) {
+    this.maxBackupIndex = maxBackupIndex;
   }
 
+  protected synchronized void cleanUp() {
+    String regex = "";
+    try {
+      File actualFile = new File(fileName);
+      
+      String directoryName = actualFile.getParent();
+      String actualFileName = actualFile.getName();
+      File dirList = new File(directoryName);
+      
+      
+      if (cleanUpRegex == null || !cleanUpRegex.contains("$fileName")) {
+        LogLog.error("cleanUpRegex == null || !cleanUpRegex.contains(\"$fileName\")");
+        return;
+      }
+      regex =cleanUpRegex.replace("$fileName", actualFileName);
+      String[] dirFiles = dirList.list(new LogFilter(actualFileName,regex));
+      
+      List<String> files = new ArrayList<String>();
+      for(String file: dirFiles) {
+        files.add(file); 
+      }
+      Collections.sort(files);
+      
+      while(files.size() > maxBackupIndex) {
+        String file = files.remove(0);
+        File f = new File(directoryName + "/" +file);
+        f.delete();
+        LogLog.debug("Removing: " +file);
+      }
+    } catch(Exception e) {
+      errorHandler.error("cleanUp("+fileName+"," + regex +") call failed.");
+    }
+  }
+  
+  private class LogFilter implements FilenameFilter {
+    private Pattern p = null;
+    private String logFile = null;
+ 
+    public LogFilter(String logFile,String regex) {
+      this.logFile = logFile;
+      p = Pattern.compile(regex); 
+    }
+ 
+    @Override
+    public boolean accept(File dir, String name) {
+      // ignore current log file
+      if (name.intern() == this.logFile.intern() ) {
+        return false;
+      }
+      //ignore file without the same prefix
+      if (!name.startsWith(logFile)) {
+        return false;
+      }
+      return p.matcher(name).find();
+    }
+  }
   
   private class ClientFinalizer extends Thread 
   {
@@ -614,3 +703,4 @@
     return getTime();
   }
 }
+



Mime
View raw message