chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r786718 - in /hadoop/chukwa: branches/chukwa-0.2/ branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/ trunk/ trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/
Date Fri, 19 Jun 2009 23:29:50 GMT
Author: eyang
Date: Fri Jun 19 23:29:50 2009
New Revision: 786718

URL: http://svn.apache.org/viewvc?rev=786718&view=rev
Log:
CHUKWA-318. Added check for disk capacity for collector to write. (Jerome Boulon via Eric
Yang)

Modified:
    hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java

Modified: hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt?rev=786718&r1=786717&r2=786718&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt (original)
+++ hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt Fri Jun 19 23:29:50 2009
@@ -34,6 +34,8 @@
 
   IMPROVEMENTS
 
+    CHUKWA-318. Added check for disk capacity for collector to write. (Jerome Boulon via
Eric Yang)
+
     CHUKWA-276. Fix hourly and daily rolling to use a single reducer. (Jerome Boulon via
Eric Yang)
 
     CHUKWA-313. Removed the 10 second ack from SeqFileWriter. (Eric Yang)

Modified: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java?rev=786718&r1=786717&r2=786718&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java
Fri Jun 19 23:29:50 2009
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.chukwa.datacollection.writer.localfs;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -112,7 +113,8 @@
 
   private long timePeriod = -1;
   private long nextTimePeriodComputation = -1;
-
+  private int minPercentFreeDisk = 20;
+  
   static {
     try {
       localHostAddr = "_" + InetAddress.getLocalHost().getHostName() + "_";
@@ -151,7 +153,7 @@
     }
 
     
-    
+    minPercentFreeDisk = conf.getInt("chukwaCollector.minPercentFreeDisk",20);
     
     rotateInterval = conf.getInt("chukwaCollector.rotateInterval",
         1000 * 60 * 5);// defaults to 5 minutes
@@ -160,11 +162,11 @@
         .getInt("chukwaCollector.writeChunkRetries", 10);
     writeChunkRetries = initWriteChunkRetries;
 
-    // check if they've told us the file system to use
     log.info("rotateInterval is " + rotateInterval);
     log.info("outputDir is " + localOutputDir);
     log.info("localFileSystem is " + fs.getUri().toString());
-
+    log.info("minPercentFreeDisk is " + minPercentFreeDisk);
+    
     // Setup everything by rotating
     rotate();
 
@@ -300,6 +302,7 @@
             fs.delete(previousPath, false);
           }
         }
+        
         Path newOutputPath = new Path(newName + ".chukwa");
         FSDataOutputStream newOutputStr = fs.create(newOutputPath);
         
@@ -320,6 +323,23 @@
       }
     }
  
+    // Check for disk space
+    File directory4Space = new File(localOutputDir);
+    long totalSpace = directory4Space.getTotalSpace();
+    long freeSpace = directory4Space.getFreeSpace();
+    long minFreeAvailable = (totalSpace * minPercentFreeDisk) /100;
+    
+    if (log.isDebugEnabled()) {
+      log.debug("Directory: " + localOutputDir + ", totalSpace: " + totalSpace 
+          + ", freeSpace: " + freeSpace + ", minFreeAvailable: " + minFreeAvailable
+          + ", percentFreeDisk: " + minPercentFreeDisk);
+    }
+  
+    if (freeSpace < minFreeAvailable) {
+      log.fatal("No space left on device, Bail out!");
+      DaemonWatcher.bailout(-1);
+    } 
+    
     log.debug("finished rotate()");
   }
 

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=786718&r1=786717&r2=786718&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Fri Jun 19 23:29:50 2009
@@ -32,6 +32,8 @@
 
   IMPROVEMENTS
 
+    CHUKWA-318. Added check for disk capacity for collector to write. (Jerome Boulon via
Eric Yang)
+
     CHUKWA-276. Fix hourly and daily rolling to use a single reducer. (Jerome Boulon via
Eric Yang)
 
     CHUKWA-313. Removed the 10 second ack from SeqFileWriter. (Eric Yang)

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java?rev=786718&r1=786717&r2=786718&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/localfs/LocalWriter.java
Fri Jun 19 23:29:50 2009
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.chukwa.datacollection.writer.localfs;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -112,7 +113,8 @@
 
   private long timePeriod = -1;
   private long nextTimePeriodComputation = -1;
-
+  private int minPercentFreeDisk = 20;
+  
   static {
     try {
       localHostAddr = "_" + InetAddress.getLocalHost().getHostName() + "_";
@@ -151,7 +153,7 @@
     }
 
     
-    
+    minPercentFreeDisk = conf.getInt("chukwaCollector.minPercentFreeDisk",20);
     
     rotateInterval = conf.getInt("chukwaCollector.rotateInterval",
         1000 * 60 * 5);// defaults to 5 minutes
@@ -160,11 +162,11 @@
         .getInt("chukwaCollector.writeChunkRetries", 10);
     writeChunkRetries = initWriteChunkRetries;
 
-    // check if they've told us the file system to use
     log.info("rotateInterval is " + rotateInterval);
     log.info("outputDir is " + localOutputDir);
     log.info("localFileSystem is " + fs.getUri().toString());
-
+    log.info("minPercentFreeDisk is " + minPercentFreeDisk);
+    
     // Setup everything by rotating
     rotate();
 
@@ -300,6 +302,7 @@
             fs.delete(previousPath, false);
           }
         }
+        
         Path newOutputPath = new Path(newName + ".chukwa");
         FSDataOutputStream newOutputStr = fs.create(newOutputPath);
         
@@ -320,6 +323,23 @@
       }
     }
  
+    // Check for disk space
+    File directory4Space = new File(localOutputDir);
+    long totalSpace = directory4Space.getTotalSpace();
+    long freeSpace = directory4Space.getFreeSpace();
+    long minFreeAvailable = (totalSpace * minPercentFreeDisk) /100;
+    
+    if (log.isDebugEnabled()) {
+      log.debug("Directory: " + localOutputDir + ", totalSpace: " + totalSpace 
+          + ", freeSpace: " + freeSpace + ", minFreeAvailable: " + minFreeAvailable
+          + ", percentFreeDisk: " + minPercentFreeDisk);
+    }
+  
+    if (freeSpace < minFreeAvailable) {
+      log.fatal("No space left on device, Bail out!");
+      DaemonWatcher.bailout(-1);
+    } 
+    
     log.debug("finished rotate()");
   }
 



Mime
View raw message