hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r915223 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Date Tue, 23 Feb 2010 07:31:08 GMT
Author: dhruba
Date: Tue Feb 23 07:31:08 2010
New Revision: 915223

URL: http://svn.apache.org/viewvc?rev=915223&view=rev
Log:
MAPREDUCE-1519. RaidNode fails to create new parity file 
if an older parity file already exists. (Rodrigo Schmidt via dhruba)


Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=915223&r1=915222&r2=915223&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Feb 23 07:31:08 2010
@@ -378,6 +378,9 @@
     MAPREDUCE-1378. URL encode link in jobhistory.jsp to avoid errors caused by
     unescaped characters. (E. Sammer via cdouglas)
 
+    MAPREDUCE-1519. RaidNode fails to create new parity file 
+    if an older version already exists. (Rodrigo Schmidt via dhruba)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=915223&r1=915222&r2=915223&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Tue Feb 23 07:31:08 2010
@@ -810,6 +810,10 @@
       out.close();
       out = null;
 
+      // delete destination if exists
+      if (outFs.exists(outpath)){
+        outFs.delete(outpath, false);
+      }
       // rename tmppath to the real parity filename
       if (!outFs.rename(tmppath, outpath)) {
         String msg = "Unable to rename tmp file " + tmppath + " to " + outpath;

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java?rev=915223&r1=915222&r2=915223&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Tue Feb 23 07:31:08 2010
@@ -334,6 +334,7 @@
       long crc2 = createOldFile(fileSys, file2, 3, numBlock, blockSize);
       FileStatus[] listPaths = null;
 
+      long firstmodtime = 0;
       // wait till file is raided
       while (true) {
         Thread.sleep(20000L);                  // waiting
@@ -344,6 +345,7 @@
             LOG.info("doCheckPolicy found path " + s.getPath());
             if (!s.getPath().toString().endsWith(".tmp")) {
               count++;
+              firstmodtime = s.getModificationTime();
             }
           }
         }
@@ -354,7 +356,39 @@
                  (listPaths == null ? "none" : listPaths.length));
       }
       assertEquals(listPaths.length, 1);
-      LOG.info("doCheckPolicy all files found in Raid.");
+
+      LOG.info("doCheckPolicy all files found in Raid the first time.");
+
+      LOG.info("doCheckPolicy: recreating source file");
+      crc2 = createOldFile(fileSys, file2, 3, numBlock, blockSize);
+
+      FileStatus st = fileSys.getFileStatus(file2);
+      assertTrue(st.getModificationTime() > firstmodtime);
+      
+      // wait till file is raided
+      while (true) {
+        Thread.sleep(20000L);                  // waiting
+        listPaths = fileSys.listStatus(destPath);
+        int count = 0;
+        if (listPaths != null && listPaths.length == 1) {
+          for (FileStatus s : listPaths) {
+            LOG.info("doCheckPolicy found path " + s.getPath() + " " + s.getModificationTime());
+            if (!s.getPath().toString().endsWith(".tmp") &&
+                s.getModificationTime() > firstmodtime) {
+              count++;
+            }
+          }
+        }
+        if (count > 0) {
+          break;
+        }
+        LOG.info("doCheckPolicy waiting for files to be raided. Found " + 
+                 (listPaths == null ? "none" : listPaths.length));
+      }
+      assertEquals(listPaths.length, 1);
+
+      LOG.info("doCheckPolicy: file got re-raided as expected.");
+      
     } catch (Exception e) {
       LOG.info("doCheckPolicy Exception " + e +
                                           StringUtils.stringifyException(e));



Mime
View raw message