hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r1003058 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/raid/README src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Date Thu, 30 Sep 2010 13:22:29 GMT
Author: dhruba
Date: Thu Sep 30 13:22:28 2010
New Revision: 1003058

URL: http://svn.apache.org/viewvc?rev=1003058&view=rev
Log:
MAPREDUCE-1816. HAR files used for RAID parity-bite have configurable 
partfile size. (Ramkumar Vadali via dhruba)


Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/raid/README
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1003058&r1=1003057&r2=1003058&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Sep 30 13:22:28 2010
@@ -316,6 +316,9 @@ Trunk (unreleased changes)
     MAPREDUCE-2029. DistributedRaidFileSystem removes itself from FileSystem
     cache when it is closed. (Ramkumar Vadali via dhruba)
 
+    MAPREDUCE-1816. HAR files used for RAID parity-bite have configurable 
+    partfile size. (Ramkumar Vadali via dhruba)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/contrib/raid/README
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/README?rev=1003058&r1=1003057&r2=1003058&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/README (original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/README Thu Sep 30 13:22:28 2010
@@ -94,6 +94,16 @@ The following properties can be set in h
           </description>
         </property>
 
+    Specify the size of HAR part-files
+        <property>
+          <name>raid.har.partfile.size</name>
+          <value>4294967296</value>
+          <description>The size of HAR part files that store raid parity
+          files. The default is 4GB. The higher the number the fewer the
+          number of files used to store the HAR archive.
+          </description>
+        </property>
+
     Specify RaidNode to not use a map-reduce cluster for raiding files in parallel.
         <property>
           <name>fs.raidnode.local</name>

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java?rev=1003058&r1=1003057&r2=1003058&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
Thu Sep 30 13:22:28 2010
@@ -53,6 +53,8 @@ class ConfigManager {
 
   /** Time to wait between successive runs of all policies */
   public static final long RESCAN_INTERVAL = 3600 * 1000;
+
+  public static final long HAR_PARTFILE_SIZE = 10 * 1024 * 1024 * 1024l;
   
   /**
    * Time to wait after the config file has been modified before reloading it
@@ -68,6 +70,7 @@ class ConfigManager {
   private boolean lastReloadAttemptFailed = false;
   private long reloadInterval = RELOAD_INTERVAL;
   private long periodicity; // time between runs of all policies
+  private long harPartfileSize;
 
   // Reload the configuration
   private boolean doReload;
@@ -84,6 +87,7 @@ class ConfigManager {
     this.doReload = conf.getBoolean("raid.config.reload", true);
     this.reloadInterval = conf.getLong("raid.config.reload.interval", RELOAD_INTERVAL);
     this.periodicity = conf.getLong("raid.policy.rescan.interval",  RESCAN_INTERVAL);
+    this.harPartfileSize = conf.getLong("raid.har.partfile.size", HAR_PARTFILE_SIZE);
     if (configFileName == null) {
       String msg = "No raid.config.file given in conf - " +
                    "the Hadoop Raid utility cannot run. Aborting....";
@@ -297,6 +301,10 @@ class ConfigManager {
   public synchronized long getPeriodicity() {
     return periodicity;
   }
+
+  public synchronized long getHarPartfileSize() {
+    return harPartfileSize;
+  }
   
   /**
    * Get a collection of all policies

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=1003058&r1=1003057&r2=1003058&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Thu Sep 30 13:22:28 2010
@@ -1446,6 +1446,8 @@ public class RaidNode implements RaidPro
     String harFileDst = qualifiedPath.getName() + HAR_SUFFIX;
     String harFileSrc = qualifiedPath.getName() + "-" + 
                                 rand.nextLong() + "-" + HAR_SUFFIX;
+    // HadoopArchives.HAR_PARTFILE_LABEL is private, so hard-coding the label.
+    conf.setLong("har.partfile.size", configMgr.getHarPartfileSize());
     HadoopArchives har = new HadoopArchives(conf);
     String[] args = new String[6];
     args[0] = "-archiveName";



Mime
View raw message