hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r956335 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
Date Sun, 20 Jun 2010 09:08:54 GMT
Author: dhruba
Date: Sun Jun 20 09:08:53 2010
New Revision: 956335

URL: http://svn.apache.org/viewvc?rev=956335&view=rev
Log:
MAPREDUCE-1492. Delete  obsolete har files used on the parity files
of hdfs raid. (Rodrigo Schmidt via dhruba)


Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=956335&r1=956334&r2=956335&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Sun Jun 20 09:08:53 2010
@@ -102,6 +102,9 @@ Trunk (unreleased changes)
     MAPREDUCE-1559. Fixes the token renewer to use the JobTracker's 
     credentials for talking to the NameNode. (ddas)
 
+    MAPREDUCE-1492. Delete  obsolete har files used on the parity files
+    of hdfs raid. (Rodrigo Schmidt via dhruba)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=956335&r1=956334&r2=956335&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Sun Jun 20 09:08:53 2010
@@ -1153,6 +1153,7 @@ public class RaidNode implements RaidPro
           Thread.sleep(SLEEP_TIME);
         }
 
+        LOG.info("Started purge scan");
         prevExec = now();
         
         // fetch all categories
@@ -1235,6 +1236,12 @@ public class RaidNode implements RaidPro
 
       // Verify if it is a har file
       if (destStr.endsWith(HAR_SUFFIX)) {
+        String destParentStr = destPath.getParent().toUri().getPath();
+        String src = destParentStr.replaceFirst(destPrefix, "");
+        Path srcPath = new Path(src);
+        if (!srcFs.exists(srcPath)) {
+          destFs.delete(destPath, true);
+        }
         return;
       }
       

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java?rev=956335&r1=956334&r2=956335&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
Sun Jun 20 09:08:53 2010
@@ -198,11 +198,11 @@ public class TestRaidHar extends TestCas
              " blockSize=" + blockSize + " stripeLength=" + stripeLength);
     mySetup(srcPath, targetReplication, metaReplication, stripeLength);
     RaidShell shell = null;
-    Path dir = new Path("/user/test/raidtest/");
+    Path dir = new Path("/user/test/raidtest/subdir/");
     Path file1 = new Path(dir + "/file" + iter);
     RaidNode cnode = null;
     try {
-      Path destPath = new Path("/destraid/user/test/raidtest");
+      Path destPath = new Path("/destraid/user/test/raidtest/subdir");
       fileSys.delete(dir, true);
       fileSys.delete(destPath, true);
       TestRaidNode.createOldFile(fileSys, file1, 1, numBlock, blockSize);
@@ -250,6 +250,26 @@ public class TestRaidHar extends TestCas
         Thread.sleep(1000);                  // keep waiting
       }
       
+      fileSys.delete(dir, true);
+      // wait till raid file is deleted
+      int count = 1;
+      while (count > 0) {
+        count = 0;
+        try {
+          listPaths = fileSys.listStatus(destPath);
+          if (listPaths != null) {
+            for (FileStatus s : listPaths) {
+              LOG.info("doTestHar found path " + s.getPath());
+              if (s.getPath().toString().endsWith(".har")) {
+                count++;
+              }
+            }
+          }
+        } catch (FileNotFoundException e) { } //ignoring
+        LOG.info("doTestHar waiting for har file to be deleted. Found " + 
+                (listPaths == null ? "none" : listPaths.length) + " files");
+        Thread.sleep(1000);
+      }
       
     } catch (Exception e) {
       LOG.info("doTestHar Exception " + e +



Mime
View raw message