hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r918206 - in /hadoop/mapreduce/trunk: ./ src/contrib/raid/src/java/org/apache/hadoop/raid/ src/contrib/raid/src/test/org/apache/hadoop/hdfs/ src/contrib/raid/src/test/org/apache/hadoop/raid/
Date Tue, 02 Mar 2010 21:47:40 GMT
Author: dhruba
Date: Tue Mar  2 21:47:39 2010
New Revision: 918206

URL: http://svn.apache.org/viewvc?rev=918206&view=rev
Log:
MAPREDUCE-1510. RAID should regenerate parity files if they get deleted.
(Rodrigo Schmidt via dhruba)


Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidPurge.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=918206&r1=918205&r2=918206&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Mar  2 21:47:39 2010
@@ -392,6 +392,9 @@
     MAPREDUCE-1537. The patch makes the job client call the getDelegationToken
     only when security is enabled. (Jitendra Nath Pandey via ddas)
 
+    MAPREDUCE-1510. RAID should regenerate parity files if they get deleted.
+    (Rodrigo Schmidt via dhruba)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=918206&r1=918205&r2=918206&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Tue Mar  2 21:47:39 2010
@@ -224,7 +224,10 @@
     return getAddress(nodeport);
   }
 
-
+  public InetSocketAddress getListenerAddress() {
+    return server.getListenerAddress();
+  }
+  
   private void initialize(Configuration conf) 
     throws IOException, SAXException, InterruptedException, RaidConfigurationException,
            ClassNotFoundException, ParserConfigurationException {
@@ -582,12 +585,6 @@
         return;
       }
 
-      // If the replication factor of the source file is already below what is
-      // expected by RAID, then ignore it.
-      if (src.getReplication() < srcReplication) {
-        return;
-      }
-
       // check if destination path already exists. If it does and it's modification time
       // does not match the modTime of the source file, then recalculate RAID
       boolean add = false;

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java?rev=918206&r1=918205&r2=918206&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java
Tue Mar  2 21:47:39 2010
@@ -74,13 +74,17 @@
    * @throws IOException
    */
   public RaidShell(Configuration conf) throws IOException {
+    this(conf, RaidNode.getAddress(conf));
+  }
+
+  public RaidShell(Configuration conf, InetSocketAddress address) throws IOException {
     super(conf);
     this.ugi = UserGroupInformation.getCurrentUser();
 
-    this.rpcRaidnode = createRPCRaidnode(RaidNode.getAddress(conf), conf, ugi);
+    this.rpcRaidnode = createRPCRaidnode(address, conf, ugi);
     this.raidnode = createRaidnode(rpcRaidnode);
   }
-
+  
   public static RaidProtocol createRaidnode(Configuration conf) throws IOException {
     return createRaidnode(RaidNode.getAddress(conf), conf);
   }

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java?rev=918206&r1=918205&r2=918206&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java
Tue Mar  2 21:47:39 2010
@@ -84,6 +84,7 @@
 
     // do not use map-reduce cluster for Raiding
     conf.setBoolean("fs.raidnode.local", true);
+    conf.set("raid.server.address", "localhost:0");
     conf.setInt("hdfs.raid.stripeLength", 3);
     conf.set("hdfs.raid.locations", "/destraid");
 

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java?rev=918206&r1=918205&r2=918206&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
Tue Mar  2 21:47:39 2010
@@ -81,6 +81,8 @@
     // the RaidNode does the raiding inline (instead of submitting to map/reduce)
     conf.setBoolean("fs.raidnode.local", local);
 
+    conf.set("raid.server.address", "localhost:0");
+
     // create a dfs and map-reduce cluster
     final int taskTrackers = 4;
 
@@ -209,10 +211,10 @@
 
       while (times-- > 0) {
         try {
-          shell = new RaidShell(conf);
+          shell = new RaidShell(conf, cnode.getListenerAddress());
         } catch (Exception e) {
-          LOG.info("doTestHar unable to connect to " + RaidNode.getAddress(conf) +
-                   " retrying....");
+          LOG.info("doTestHar unable to connect to " + 
+              cnode.getListenerAddress() + " retrying....");
           Thread.sleep(1000);
           continue;
         }

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java?rev=918206&r1=918205&r2=918206&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Tue Mar  2 21:47:39 2010
@@ -85,6 +85,8 @@
     // the RaidNode does the raiding inline (instead of submitting to map/reduce)
     conf.setBoolean("fs.raidnode.local", local);
 
+    conf.set("raid.server.address", "localhost:0");
+    
     // create a dfs and map-reduce cluster
     final int taskTrackers = 4;
     final int jobTrackerPort = 60050;
@@ -209,10 +211,10 @@
 
       while (times-- > 0) {
         try {
-          shell = new RaidShell(conf);
+          shell = new RaidShell(conf, cnode.getListenerAddress());
         } catch (Exception e) {
-          LOG.info("doTestPathFilter unable to connect to " + RaidNode.getAddress(conf) +
-                   " retrying....");
+          LOG.info("doTestPathFilter unable to connect to " + 
+              cnode.getListenerAddress() + " retrying....");
           Thread.sleep(1000);
           continue;
         }
@@ -308,21 +310,16 @@
       fileSys.delete(dir, true);
       fileSys.delete(destPath, true);
 
-      // create a file that has a replciation factor of 2. The Raid code should not
-      // pick this up.
-      long crc1 = createOldFile(fileSys, file1, 2, numBlock, blockSize);
-      LOG.info("doCheckPolicy created test files.");
-
       // create an instance of the RaidNode
       cnode = RaidNode.createRaidNode(null, conf);
       int times = 10;
 
       while (times-- > 0) {
         try {
-          shell = new RaidShell(conf);
+          shell = new RaidShell(conf, cnode.getListenerAddress());
         } catch (Exception e) {
-          LOG.info("doCheckPolicy unable to connect to " + RaidNode.getAddress(conf) +
-                   " retrying....");
+          LOG.info("doCheckPolicy unable to connect to " + 
+              cnode.getListenerAddress() + " retrying....");
           Thread.sleep(1000);
           continue;
         }
@@ -331,7 +328,7 @@
       LOG.info("doCheckPolicy created RaidShell.");
 
       // this file should be picked up RaidNode
-      long crc2 = createOldFile(fileSys, file2, 3, numBlock, blockSize);
+      long crc2 = createOldFile(fileSys, file2, 2, numBlock, blockSize);
       FileStatus[] listPaths = null;
 
       long firstmodtime = 0;
@@ -360,7 +357,7 @@
       LOG.info("doCheckPolicy all files found in Raid the first time.");
 
       LOG.info("doCheckPolicy: recreating source file");
-      crc2 = createOldFile(fileSys, file2, 3, numBlock, blockSize);
+      crc2 = createOldFile(fileSys, file2, 2, numBlock, blockSize);
 
       FileStatus st = fileSys.getFileStatus(file2);
       assertTrue(st.getModificationTime() > firstmodtime);

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidPurge.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidPurge.java?rev=918206&r1=918205&r2=918206&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidPurge.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidPurge.java
Tue Mar  2 21:47:39 2010
@@ -91,6 +91,8 @@
     // the RaidNode does the raiding inline (instead of submitting to map/reduce)
     conf.setBoolean("fs.raidnode.local", local);
 
+    conf.set("raid.server.address", "localhost:0");
+    
     // create a dfs and map-reduce cluster
     final int taskTrackers = 4;
     final int jobTrackerPort = 60050;
@@ -209,10 +211,10 @@
 
       while (times-- > 0) {
         try {
-          shell = new RaidShell(conf);
+          shell = new RaidShell(conf, cnode.getListenerAddress());
         } catch (Exception e) {
-          LOG.info("doTestPurge unable to connect to " + RaidNode.getAddress(conf) +
-                   " retrying....");
+          LOG.info("doTestPurge unable to connect to " + 
+              cnode.getListenerAddress() + " retrying....");
           Thread.sleep(1000);
           continue;
         }



Mime
View raw message