hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sc...@apache.org
Subject svn commit: r1067049 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java
Date Fri, 04 Feb 2011 00:43:51 GMT
Author: schen
Date: Fri Feb  4 00:43:51 2011
New Revision: 1067049

URL: http://svn.apache.org/viewvc?rev=1067049&view=rev
Log:
MAPREDUCE-1706. Log RAID recoveries on HDFS. (schen)

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1067049&r1=1067048&r2=1067049&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Feb  4 00:43:51 2011
@@ -27,6 +27,8 @@ Trunk (unreleased changes)
     MAPREDUCE-2263. MapReduce side of HADOOP-6904: RPC compatibility.
     (hairong)
 
+    MAPREDUCE-1706. Log RAID recoveries on HDFS. (schen)
+
   OPTIMIZATIONS
     
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java?rev=1067049&r1=1067048&r2=1067049&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/hdfs/DistributedRaidFileSystem.java
Fri Feb  4 00:43:51 2011
@@ -17,37 +17,28 @@
  */
 package org.apache.hadoop.hdfs;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.DataInput;
 import java.io.PrintStream;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.ChecksumException;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.hdfs.BlockMissingException;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.raid.Decoder;
 import org.apache.hadoop.raid.RaidNode;
 import org.apache.hadoop.raid.ReedSolomonDecoder;
 import org.apache.hadoop.raid.XORDecoder;
 import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType;
+import org.apache.hadoop.util.ReflectionUtils;
 
 /**
  * This is an implementation of the Hadoop  RAID Filesystem. This FileSystem 
@@ -445,6 +436,29 @@ public class DistributedRaidFileSystem e
                          stripeLength, corruptOffset);
             if (npath == null)
               continue;
+            try {
+              String outdir = conf.get("fs.raid.recoverylogdir");
+              if (outdir != null) {
+                DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
+                java.util.Date date = new java.util.Date();
+                String fname = path.getName() + dateFormat.format(date) +
+                (new Random()).nextInt() + ".txt";
+                Path outputunraid = new Path(outdir, fname);
+                FileSystem fs = outputunraid.getFileSystem(conf);
+                FSDataOutputStream dout = fs.create(outputunraid);
+                PrintStream ps = new PrintStream(dout);
+                ps.println("Recovery attempt log");
+                ps.println("Source path : " + path );
+                ps.println("Alternate path : " + alternates[idx].destPath);
+                ps.println("Stripe lentgh : " + stripeLength);
+                ps.println("Corrupt offset : " + corruptOffset);
+                String output = (npath==null) ? "UNSUCCESSFUL" : npath.toString();
+                ps.println("Output from unRaid : " + output);
+                ps.close();
+              }
+            } catch (Exception exc) {
+              LOG.info("Error while creating recovery log: " + exc);
+            }
 
             closeCurrentStream();
             LOG.info("Using block at offset " + corruptOffset + " from " +

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java?rev=1067049&r1=1067048&r2=1067049&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/TestRaidDfs.java
Fri Feb  4 00:43:51 2011
@@ -17,49 +17,38 @@
  */
 package org.apache.hadoop.hdfs;
 
+import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileWriter;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.InputStreamReader;
 import java.io.RandomAccessFile;
 import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.GregorianCalendar;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Properties;
 import java.util.Random;
+import java.util.regex.Pattern;
 import java.util.zip.CRC32;
 
 import junit.framework.TestCase;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FilterFileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.DistributedRaidFileSystem;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.raid.RaidNode;
 import org.apache.hadoop.raid.RaidUtils;
 import org.apache.hadoop.raid.protocol.PolicyInfo.ErasureCodeType;
+import org.apache.hadoop.util.StringUtils;
 
 public class TestRaidDfs extends TestCase {
   final static String TEST_DIR = new File(System.getProperty("test.build.data",
       "build/contrib/raid/test/data")).getAbsolutePath();
+  final static String LOG_DIR = "/raidlog";
   final static long RELOAD_INTERVAL = 1000;
   final static Log LOG = LogFactory.getLog("org.apache.hadoop.raid.TestRaidDfs");
   final static int NUM_DATANODES = 3;
@@ -79,6 +68,7 @@ public class TestRaidDfs extends TestCas
     new File(TEST_DIR).mkdirs(); // Make sure data directory exists
     conf = new Configuration();
 
+    conf.set("fs.raid.recoverylogdir", LOG_DIR);
     conf.setInt(RaidNode.RS_PARITY_LENGTH_KEY, rsParityLength);
 
     // scan all policies once every 5 second
@@ -198,6 +188,7 @@ public class TestRaidDfs extends TestCas
     // Validate
     DistributedRaidFileSystem raidfs = getRaidFS();
     assertTrue(validateFile(raidfs, srcFile, length, crc));
+    validateLogFile(getRaidFS(), new Path(LOG_DIR));
   }
 
   /**
@@ -429,6 +420,30 @@ public class TestRaidDfs extends TestCas
     }
     return true;
   }
+  //
+  // validates the contents of raid recovery log file
+  //
+  public static void validateLogFile(FileSystem fileSys, Path logDir)
+      throws IOException {
+    FileStatus f = fileSys.listStatus(logDir)[0];
+    FSDataInputStream stm = fileSys.open(f.getPath());
+    try {
+      BufferedReader reader = new BufferedReader(new InputStreamReader(stm));
+      assertEquals("Recovery attempt log", reader.readLine());
+      assertTrue(Pattern.matches("Source path : /user/dhruba/raidtest/.*",
+                 reader.readLine()));
+      assertTrue(Pattern.matches("Alternate path : .*/destraid",
+                 reader.readLine()));
+      assertEquals("Stripe lentgh : 3", reader.readLine());
+      assertTrue(Pattern.matches("Corrupt offset : \\d*", reader.readLine()));
+      assertTrue(Pattern.matches("Output from unRaid : " +
+          "hdfs://.*/tmp/raid/user/dhruba/raidtest/.*recovered",
+          reader.readLine()));
+    } finally {
+      stm.close();
+    }
+    LOG.info("Raid HDFS Recovery log verified");
+  }
 
   /*
    * The Data directories for a datanode



Mime
View raw message