hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r990684 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Date Mon, 30 Aug 2010 01:46:37 GMT
Author: dhruba
Date: Mon Aug 30 01:46:37 2010
New Revision: 990684

URL: http://svn.apache.org/viewvc?rev=990684&view=rev
Log:
MAPREDUCE-1670. RAID policies should not scan their own destination path.
(Ramkumar Vadali via dhruba)


Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=990684&r1=990683&r2=990684&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Aug 30 01:46:37 2010
@@ -259,6 +259,9 @@ Trunk (unreleased changes)
 
     MAPREDUCE-2022. Fixes compilation errors in TestSubmitJob. (amareshwari)
 
+    MAPREDUCE-1670. RAID policies should not scan their own destination path.
+    (Ramkumar Vadali via dhruba)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=990684&r1=990683&r2=990684&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Mon Aug 30 01:46:37 2010
@@ -561,7 +561,7 @@ public class RaidNode implements RaidPro
  /**
   * Returns a list of pathnames that needs raiding.
   */
-  private List<FileStatus> selectFiles(Configuration conf, Path p, String destPrefix,
+  List<FileStatus> selectFiles(Configuration conf, Path p, String destPrefix,
                                        long modTimePeriod, short srcReplication, long now)
throws IOException {
 
     List<FileStatus> returnSet = new LinkedList<FileStatus>();
@@ -571,11 +571,26 @@ public class RaidNode implements RaidPro
     FileSystem fs = FileSystem.get(destp.toUri(), conf);
     destp = destp.makeQualified(fs);
 
+    // Expand destination prefix path.
+    String destpstr = destp.toString();
+    if (!destpstr.endsWith(Path.SEPARATOR)) {
+      destpstr += Path.SEPARATOR;
+    }
+
     fs = p.getFileSystem(conf);
     FileStatus[] gpaths = fs.globStatus(p);
-    if (gpaths != null){
+    if (gpaths != null) {
       for (FileStatus onepath: gpaths) {
-        recurse(fs, conf, destp, onepath, returnSet, modTimePeriod, srcReplication, now);
+        String pathstr = onepath.getPath().makeQualified(fs).toString();
+        if (!pathstr.endsWith(Path.SEPARATOR)) {
+          pathstr += Path.SEPARATOR;
+        }
+        if (pathstr.startsWith(destpstr) || destpstr.startsWith(pathstr)) {
+          LOG.info("Skipping source " + pathstr +
+                   " because it conflicts with raid directory " + destpstr);
+        } else {
+         recurse(fs, conf, destp, onepath, returnSet, modTimePeriod, srcReplication, now);
+        }
       }
     }
     return returnSet;

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java?rev=990684&r1=990683&r2=990684&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Mon Aug 30 01:46:37 2010
@@ -450,6 +450,59 @@ public class TestRaidNode extends TestCa
     LOG.info("Test testDistRaid completed.");
   }
   
+  /**
+   * Test the case where the source and destination paths conflict.
+   * @throws Exception
+   */
+  public void testConflictingPaths() throws Exception {
+    LOG.info("Test testConflictingPaths started");
+    long targetReplication = 2;
+    long metaReplication   = 2;
+    long stripeLength      = 3;
+    short srcReplication = 1;
+    long modTimePeriod = 0;
+    try {
+      createClusters(false);
+      mySetup("/user/d/raidtest", srcReplication, targetReplication,
+          metaReplication, stripeLength);
+      // We dont need this to run, just need the object.
+      RaidNode cnode = RaidNode.createRaidNode(null, conf);
+      cnode.stop();
+      cnode.join();
+
+      createOldFile(fileSys, new Path("/user/d/raidtest/f1"), 2, 7, 8192L);
+      LOG.info("Test testConflictingPaths created test files");
+
+      long now = System.currentTimeMillis();
+
+      // Test the regular case.
+      LOG.info("Test testConflictingPaths testing the regular case");
+      List<FileStatus> selected = cnode.selectFiles(conf,
+          new Path("/user/d/raidtest*"), "/raid",
+          modTimePeriod, srcReplication, now);
+      assertTrue(selected.size() > 0);
+
+      // Test the conflicting case: src under dest.
+      LOG.info("Test testConflictingPaths testing src under dest");
+      selected = cnode.selectFiles(conf,
+          new Path("/user/d/raidtest*"), "/user/d",
+          modTimePeriod, srcReplication, now);
+      assertEquals(0, selected.size());
+
+      // Test the conflicting case: dest under src.
+      LOG.info("Test testConflictingPaths testing dest under src");
+      selected = cnode.selectFiles(conf,
+          new Path("/user/d*"), "/user/d/raidtest",
+          modTimePeriod, srcReplication, now);
+      assertEquals(0, selected.size());
+
+      LOG.info("Test testConflictingPaths succeeded.");
+    } finally {
+      stopClusters();
+    }
+    LOG.info("Test testConflictingPaths completed.");
+  }
+
   //
   // simulate a corruption at specified offset and verify that eveyrthing is good
   //



Mime
View raw message