hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sc...@apache.org
Subject svn commit: r1063053 - in /hadoop/mapreduce/trunk: ./ src/contrib/raid/src/java/org/apache/hadoop/raid/ src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/namenode/
Date Mon, 24 Jan 2011 23:08:01 GMT
Author: schen
Date: Mon Jan 24 23:08:00 2011
New Revision: 1063053

URL: http://svn.apache.org/viewvc?rev=1063053&view=rev
Log:
MAPREDUCE-2250. Fix logging in raid code. (Ramkumar Vadali via schen)


Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/Decoder.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaidNode.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalRaidNode.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRaid.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Jan 24 23:08:00 2011
@@ -228,6 +228,8 @@ Release 0.22.0 - Unreleased
 
     MAPREDUCE-2180. Add coverage of fair scheduler servlet to system test (todd)
 
+    MAPREDUCE-2250. Fix logging in raid code. (Ramkumar Vadali via schen)
+
   OPTIMIZATIONS
 
     MAPREDUCE-1354. Enhancements to JobTracker for better performance and

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/ConfigManager.java
Mon Jan 24 23:08:00 2011
@@ -303,7 +303,7 @@ class ConfigManager {
                 curr.setProperty(pname,pvalue);
               }
             } else {
-              LOG.info("Found bad property " + propertyName +
+              LOG.warn("Found bad property " + propertyName +
                        " for srcPath" + srcPathPrefix +
                        " policy name " + policyName +
                        ". Ignoring."); 

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/Decoder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/Decoder.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/Decoder.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/Decoder.java Mon
Jan 24 23:08:00 2011
@@ -135,10 +135,10 @@ public abstract class Decoder {
           LOG.info("Copied till " + out.getPos() + " from " + srcFile);
           continue;
         } catch (BlockMissingException e) {
-          LOG.info("Encountered BME at " + srcFile + ":" + offset);
+          LOG.warn("Encountered BME at " + srcFile + ":" + offset);
           bytesAlreadyCopied = out.getPos() - offset;
         } catch (ChecksumException e) {
-          LOG.info("Encountered CE at " + srcFile + ":" + offset);
+          LOG.warn("Encountered CE at " + srcFile + ":" + offset);
           bytesAlreadyCopied = out.getPos() - offset;
         }
       }
@@ -155,7 +155,7 @@ public abstract class Decoder {
       fs.setTimes(decodedFile, srcStat.getModificationTime(),
                   srcStat.getAccessTime());
     } catch (Exception exc) {
-      LOG.info("Didn't manage to copy meta information because of " + exc +
+      LOG.warn("Didn't manage to copy meta information because of " + exc +
                " Ignoring...");
     }
 

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistBlockFixer.java
Mon Jan 24 23:08:00 2011
@@ -224,7 +224,7 @@ public class DistBlockFixer extends Bloc
    */
   private void failJob(Job job) throws IOException {
     // assume no files have been fixed
-    LOG.info("job " + job.getJobID() + "(" + job.getJobName() +
+    LOG.error("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() +
       ") finished (failed)");
     for (CorruptFileInfo fileInfo: jobIndex.get(job)) {
       fileInfo.fail();
@@ -237,7 +237,7 @@ public class DistBlockFixer extends Bloc
    */ 
   private void succeedJob(Job job, long filesSucceeded, long filesFailed)
     throws IOException {
-    LOG.info("job " + job.getJobID() + "(" + job.getJobName() +
+    LOG.info("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() +
       ") finished (succeeded)");
 
     if (filesFailed == 0) {
@@ -381,6 +381,9 @@ public class DistBlockFixer extends Bloc
     SequenceFileOutputFormat.setOutputPath(job, outDir);
 
     job.submit();
+    LOG.info("DistBlockFixer job " + job.getJobID() + "(" + job.getJobName() +
+          ") started");
+
     // submit the job before inserting it into the index
     // this way, if submit fails, we won't have added anything to the index
     insertJob(job, filesInJob);
@@ -511,7 +514,7 @@ public class DistBlockFixer extends Bloc
         LOG.error("trying to remove file not in file index: " +
                   file.toString());
       } else {
-        LOG.info("fixing " + file.toString() + " failed");
+        LOG.error("fixing " + file.toString() + " failed");
       }
       pendingFiles--;
     }

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
Mon Jan 24 23:08:00 2011
@@ -188,7 +188,7 @@ public class DistRaid extends Configured
         String s = "FAIL: " + policy + ", " + key + " "
             + StringUtils.stringifyException(e);
         context.write(new Text(key), new Text(s));
-        LOG.info(s);
+        LOG.error(s);
       } finally {
         context.setStatus(getCountString());
       }
@@ -289,7 +289,7 @@ public class DistRaid extends Configured
          if (runningJob.isSuccessful()) {
            LOG.info("Job Complete(Succeeded): " + jobID);
          } else {
-           LOG.info("Job Complete(Failed): " + jobID);
+           LOG.error("Job Complete(Failed): " + jobID);
          }
          raidPolicyPathPairList.clear();
          return true;

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaidNode.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaidNode.java
Mon Jan 24 23:08:00 2011
@@ -49,8 +49,6 @@ public class DistRaidNode extends RaidNo
     this.jobMonitor = new JobMonitor(conf);
     this.jobMonitorThread = new Daemon(this.jobMonitor);
     this.jobMonitorThread.start();
-    
-    LOG.info("created");
   }
 
   /**
@@ -105,4 +103,4 @@ public class DistRaidNode extends RaidNo
   }
 
 
-}
\ No newline at end of file
+}

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalRaidNode.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalRaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/LocalRaidNode.java
Mon Jan 24 23:08:00 2011
@@ -40,8 +40,6 @@ public class LocalRaidNode extends RaidN
 
   public LocalRaidNode(Configuration conf) throws IOException {
     super(conf);
-
-    LOG.info("created");
   }
 
   /**
@@ -59,4 +57,4 @@ public class LocalRaidNode extends RaidN
   int getRunningJobsForPolicy(String policyName) {
     return 0;
   }
-}
\ No newline at end of file
+}

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
Mon Jan 24 23:08:00 2011
@@ -522,7 +522,7 @@ public abstract class RaidNode implement
           try {
             filteredPaths = selectFiles(info, allPolicies);
           } catch (Exception e) {
-            LOG.info("Exception while invoking filter on policy " + info.getName() +
+            LOG.error("Exception while invoking filter on policy " + info.getName() +
                      " srcPath " + info.getSrcPath() + 
                      " exception " + StringUtils.stringifyException(e));
             continue;
@@ -539,7 +539,7 @@ public abstract class RaidNode implement
           try {
             raidFiles(info, filteredPaths);
           } catch (Exception e) {
-            LOG.info("Exception while invoking action on policy " + info.getName() +
+            LOG.error("Exception while invoking action on policy " + info.getName() +
                      " srcPath " + info.getSrcPath() + 
                      " exception " + StringUtils.stringifyException(e));
             continue;
@@ -769,7 +769,7 @@ public abstract class RaidNode implement
     // reduce the replication factor of the source file
     if (!doSimulate) {
       if (srcFs.setReplication(p, (short)targetRepl) == false) {
-        LOG.info("Error in reducing relication factor of file " + p + " to " + targetRepl);
+        LOG.error("Error in reducing relication factor of file " + p + " to " + targetRepl);
         statistics.remainingSize += diskSpace;  // no change in disk space usage
         return;
       }
@@ -1048,7 +1048,7 @@ public abstract class RaidNode implement
             LOG.info("Purging " + destStr + " at usage " + harUsedPercent);
             boolean done = destFs.delete(destPath, true);
             if (!done) {
-              LOG.error("Could not purge " + destPath);
+              LOG.error("Unable to purge directory " + destPath);
             }
           }
         } catch (IOException e) {
@@ -1075,7 +1075,7 @@ public abstract class RaidNode implement
             LOG.info("Purged directory " + destPath );
           }
           else {
-            LOG.info("Unable to purge directory " + destPath);
+            LOG.error("Unable to purge directory " + destPath);
           }
         }
         if (files != null) {
@@ -1128,7 +1128,7 @@ public abstract class RaidNode implement
         if (done) {
           LOG.info("Purged file " + destPath );
         } else {
-          LOG.info("Unable to purge file " + destPath );
+          LOG.error("Unable to purge file " + destPath );
         }
       }
     } 
@@ -1339,7 +1339,7 @@ public abstract class RaidNode implement
       ret = ToolRunner.run(har, args);
       if (ret == 0 && !destFs.rename(new Path(tmpHarPath+"/"+harFileSrc), 
                                      new Path(qualifiedPath, harFileDst))) {
-        LOG.info("HAR rename didn't succeed from " + tmpHarPath+"/"+harFileSrc +
+        LOG.error("HAR rename didn't succeed from " + tmpHarPath+"/"+harFileSrc +
             " to " + qualifiedPath + "/" + harFileDst);
         ret = -2;
       }

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidShell.java
Mon Jan 24 23:08:00 2011
@@ -107,7 +107,7 @@ public class RaidShell extends Configure
   private static RaidProtocol createRPCRaidnode(InetSocketAddress raidNodeAddr,
       Configuration conf, UserGroupInformation ugi)
     throws IOException {
-    LOG.info("RaidShell connecting to " + raidNodeAddr);
+    LOG.debug("RaidShell connecting to " + raidNodeAddr);
     return (RaidProtocol)RPC.getProxy(RaidProtocol.class,
         RaidProtocol.versionID, raidNodeAddr, ugi, conf,
         NetUtils.getSocketFactory(conf, RaidProtocol.class));
@@ -296,9 +296,9 @@ public class RaidShell extends Configure
     for (int i = startindex; i < argv.length; i = i + 2) {
       String path = argv[i];
       long corruptOffset = Long.parseLong(argv[i+1]);
-      LOG.info("RaidShell recoverFile for " + path + " corruptOffset " + corruptOffset);
+      LOG.debug("RaidShell recoverFile for " + path + " corruptOffset " + corruptOffset);
       paths[j] = new Path(raidnode.recoverFile(path, corruptOffset));
-      LOG.info("Raidshell created recovery file " + paths[j]);
+      LOG.debug("Raidshell created recovery file " + paths[j]);
       j++;
     }
     return paths;
@@ -315,7 +315,7 @@ public class RaidShell extends Configure
 
   public void recoverBlocks(String[] args, int startIndex)
     throws IOException {
-    LOG.info("Recovering blocks for " + (args.length - startIndex) + " files");
+    LOG.debug("Recovering blocks for " + (args.length - startIndex) + " files");
     BlockFixer.BlockFixerHelper fixer = new BlockFixer.BlockFixerHelper(conf);
     for (int i = startIndex; i < args.length; i++) {
       String path = args[i];
@@ -361,11 +361,11 @@ public class RaidShell extends Configure
           corruptBlocksPerStripe.put(stripe, corruptBlocksPerStripe.
                                      get(stripe) + 1);
         }
-        LOG.info("file " + filePath.toString() + " corrupt in block " +
+        LOG.debug("file " + filePath.toString() + " corrupt in block " +
                  blockNo + "/" + fileLengthInBlocks + ", stripe " + stripe +
                  "/" + fileStripes);
       } else {
-        LOG.info("file " + filePath.toString() + " OK in block " + blockNo +
+        LOG.debug("file " + filePath.toString() + " OK in block " + blockNo +
                  "/" + fileLengthInBlocks + ", stripe " + stripe + "/" +
                  fileStripes);
       }
@@ -489,7 +489,7 @@ public class RaidShell extends Configure
                               parityBlockSize);
       }
     } else if (parityFS instanceof HarFileSystem) {
-      LOG.info("HAR FS found");
+      LOG.debug("HAR FS found");
     } else {
       LOG.warn("parity file system is not of a supported type");
     }
@@ -527,7 +527,7 @@ public class RaidShell extends Configure
                               "multiple of parity block size");
       }
       int blocksInContainer = (int) (cb.getLength() / blockSize);
-      LOG.info("found container with offset " + cb.getOffset() +
+      LOG.debug("found container with offset " + cb.getOffset() +
                ", length " + cb.getLength());
 
       for (long offset = cb.getOffset();
@@ -550,7 +550,7 @@ public class RaidShell extends Configure
 
         if (cb.isCorrupt() ||
             (cb.getNames().length == 0 && cb.getLength() > 0)) {
-          LOG.info("parity file for " + filePath.toString() +
+          LOG.debug("parity file for " + filePath.toString() +
                    " corrupt in block " + block +
                    ", stripe " + stripe + "/" + fileStripes);
 
@@ -562,7 +562,7 @@ public class RaidShell extends Configure
                                        1);
           }
         } else {
-          LOG.info("parity file for " + filePath.toString() +
+          LOG.debug("parity file for " + filePath.toString() +
                    " OK in block " + block +
                    ", stripe " + stripe + "/" + fileStripes);
         }
@@ -606,7 +606,7 @@ public class RaidShell extends Configure
     if (!rsPrefix.endsWith("/")) {
       rsPrefix = rsPrefix + "/";
     }
-    LOG.info("prefixes: " + xorPrefix + ", " + rsPrefix);
+    LOG.debug("prefixes: " + xorPrefix + ", " + rsPrefix);
 
     // get a list of corrupted files (not considering parity blocks just yet)
     // from the name node

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRaid.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRaid.java?rev=1063053&r1=1063052&r2=1063053&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRaid.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRaid.java
Mon Jan 24 23:08:00 2011
@@ -117,8 +117,8 @@ public class TestBlockPlacementPolicyRai
       FileStatus parityStat = fs.getFileStatus(new Path(parity));
       BlockLocation[] parityLoc =
           fs.getFileBlockLocations(parityStat, 0, parityStat.getLen());
-      int stripeLen = RaidNode.getStripeLength(conf);
-      for (int i = 0; i < numBlocks / stripeLen; i++) {
+      int parityLen = RaidNode.rsParityLength(conf);
+      for (int i = 0; i < numBlocks / parityLen; i++) {
         Set<String> locations = new HashSet<String>();
         for (int j = 0; j < srcLoc.length; j++) {
           String [] names = srcLoc[j].getNames();
@@ -127,8 +127,8 @@ public class TestBlockPlacementPolicyRai
             locations.add(names[k]);
           }
         }
-        for (int j = 0 ; j < stripeLen; j++) {
-          String[] names = parityLoc[j + i * stripeLen].getNames();
+        for (int j = 0 ; j < parityLen; j++) {
+          String[] names = parityLoc[j + i * parityLen].getNames();
           for (int k = 0; k < names.length; k++) {
             LOG.info("Parity block location: " + names[k]);
             Assert.assertTrue(locations.add(names[k]));



Mime
View raw message