hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject svn commit: r885924 - in /hadoop/mapreduce/branches/MAPREDUCE-233: ./ conf/ src/c++/ src/contrib/ src/contrib/block_forensics/ src/contrib/capacity-scheduler/ src/contrib/data_join/ src/contrib/dynamic-scheduler/ src/contrib/eclipse-plugin/ src/contrib...
Date Tue, 01 Dec 2009 20:51:39 GMT
Author: stevel
Date: Tue Dec  1 20:51:36 2009
New Revision: 885924

URL: http://svn.apache.org/viewvc?rev=885924&view=rev
Log:
MAPREDUCE-233 HDFS-326 sync with SVN_HEAD

Modified:
    hadoop/mapreduce/branches/MAPREDUCE-233/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/.gitignore   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/CHANGES.txt   (contents, props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/conf/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/conf/capacity-scheduler.xml.template   (props
changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/c++/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/block_forensics/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/build-contrib.xml   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/build.xml   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/capacity-scheduler/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/data_join/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/dynamic-scheduler/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/eclipse-plugin/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/fairscheduler/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/index/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/mrunit/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/sqoop/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/streaming/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/vaidya/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/examples/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/java/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/   (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/fs/   (props
changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/hdfs/   (props
changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/io/FileBench.java
  (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
  (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/ipc/   (props
changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
  (props changed)
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/tools/org/apache/hadoop/tools/DistCp.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/   (props changed)

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/mapred:713112
-/hadoop/mapreduce/trunk:804974-884916
+/hadoop/mapreduce/trunk:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,4 +1,4 @@
 /hadoop/core/branches/HADOOP-4687/mapred/.gitignore:776175-784965
 /hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
 /hadoop/core/trunk/.gitignore:784664-785643
-/hadoop/mapreduce/trunk/.gitignore:804974-884916
+/hadoop/mapreduce/trunk/.gitignore:804974-885774

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/CHANGES.txt?rev=885924&r1=885923&r2=885924&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/CHANGES.txt Tue Dec  1 20:51:36 2009
@@ -51,6 +51,9 @@
     MAPREDUCE-1169. Improvements to mysqldump use in Sqoop.
     (Aaron Kimball via tomwhite)
 
+    MAPREDUCE-1231. Added a new DistCp option, -skipcrccheck, so that the CRC
+    check during setup can be skipped.  (Jothi Padmanabhan via szetszwo)
+
   OPTIMIZATIONS
 
     MAPREDUCE-270. Fix the tasktracker to optionally send an out-of-band
@@ -94,6 +97,10 @@
    
     MAPREDUCE-1244. Fix eclipse-plugin's build dependencies. (gkesavan)
 
+    MAPREDUCE-1140. Fix DistributedCache to not decrement reference counts for
+    unreferenced files in error conditions.
+    (Amareshwari Sriramadasu via yhemanth)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
 /hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
-/hadoop/mapreduce/trunk/CHANGES.txt:804974-884916
+/hadoop/mapreduce/trunk/CHANGES.txt:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/conf:713112
 /hadoop/core/trunk/conf:784664-785643
-/hadoop/mapreduce/trunk/conf:804974-884916
+/hadoop/mapreduce/trunk/conf:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/conf/capacity-scheduler.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/conf/capacity-scheduler.xml.template:713112
 /hadoop/core/trunk/conf/capacity-scheduler.xml.template:776175-785643
-/hadoop/mapreduce/trunk/conf/capacity-scheduler.xml.template:804974-884916
+/hadoop/mapreduce/trunk/conf/capacity-scheduler.xml.template:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++:713112
 /hadoop/core/trunk/src/c++:776175-784663
-/hadoop/mapreduce/trunk/src/c++:804974-884916
+/hadoop/mapreduce/trunk/src/c++:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
 /hadoop/core/trunk/src/contrib:784664-785643
-/hadoop/mapreduce/trunk/src/contrib:804974-884916
+/hadoop/mapreduce/trunk/src/contrib:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/block_forensics/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112
-/hadoop/mapreduce/trunk/src/contrib/block_forensics:807679-884916
+/hadoop/mapreduce/trunk/src/contrib/block_forensics:807679-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
 /hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/build-contrib.xml:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/build-contrib.xml:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
 /hadoop/core/trunk/src/contrib/build.xml:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/build.xml:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/build.xml:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/capacity-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/capacity-scheduler:713112
 /hadoop/core/trunk/src/contrib/capacity-scheduler:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
 /hadoop/core/trunk/src/contrib/data_join:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/data_join:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/data_join:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/dynamic-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/src/contrib/dynamic-scheduler:713112
 /hadoop/core/trunk/src/contrib/dynamic-scheduler:784975-786373
-/hadoop/mapreduce/trunk/src/contrib/dynamic-scheduler:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/dynamic-scheduler:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
 /hadoop/core/trunk/src/contrib/eclipse-plugin:776175-784663
-/hadoop/mapreduce/trunk/src/contrib/eclipse-plugin:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/eclipse-plugin:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/fairscheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/fairscheduler:713112
 /hadoop/core/trunk/src/contrib/fairscheduler:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/fairscheduler:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/fairscheduler:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
 /hadoop/core/trunk/src/contrib/index:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/index:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/index:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/mrunit/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/mrunit:713112
 /hadoop/core/trunk/src/contrib/mrunit:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/mrunit:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/mrunit:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/sqoop/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/src/contrib/sqoop:713112
 /hadoop/core/trunk/src/contrib/sqoop:784975-786373
-/hadoop/mapreduce/trunk/src/contrib/sqoop:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/sqoop:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/streaming/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/streaming:713112
 /hadoop/core/trunk/src/contrib/streaming:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/streaming:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/streaming:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
 /hadoop/core/trunk/src/contrib/vaidya:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/vaidya:804974-884916
+/hadoop/mapreduce/trunk/src/contrib/vaidya:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/examples:713112
 /hadoop/core/trunk/src/examples:776175-784663
-/hadoop/mapreduce/trunk/src/examples:804974-884916
+/hadoop/mapreduce/trunk/src/examples:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/java:713112
 /hadoop/core/trunk/src/mapred:776175-785643
-/hadoop/mapreduce/trunk/src/java:804974-884916
+/hadoop/mapreduce/trunk/src/java:804974-885774

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java?rev=885924&r1=885923&r2=885924&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TaskDistributedCacheManager.java
Tue Dec  1 20:51:36 2009
@@ -32,7 +32,6 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
-import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalDirAllocator;
@@ -72,6 +71,7 @@
     final long timestamp;
     /** Whether this is to be added to the classpath */
     final boolean shouldBeAddedToClassPath;
+    boolean localized = false;
 
     private CacheFile(URI uri, FileType type, long timestamp, 
         boolean classPath) {
@@ -108,6 +108,14 @@
       }
       return ret;
     }
+    
+    boolean getLocalized() {
+      return localized;
+    }
+    
+    void setLocalized(boolean val) {
+      localized = val;
+    }
   }
 
   TaskDistributedCacheManager(
@@ -156,6 +164,7 @@
           cacheSubdir, fileStatus, 
           cacheFile.type == CacheFile.FileType.ARCHIVE,
           cacheFile.timestamp, workdirPath, false);
+      cacheFile.setLocalized(true);
 
       if (cacheFile.type == CacheFile.FileType.ARCHIVE) {
         localArchives.add(p);
@@ -179,6 +188,13 @@
 
   }
 
+  /*
+   * This method is called from unit tests.
+   */
+  List<CacheFile> getCacheFiles() {
+    return cacheFiles;
+  }
+  
   private static String stringifyPathList(List<Path> p){
     if (p == null || p.isEmpty()) {
       return null;
@@ -210,7 +226,9 @@
    */
   public void release() throws IOException {
     for (CacheFile c : cacheFiles) {
-      distributedCacheManager.releaseCache(c.uri, taskConf, c.timestamp);
+      if (c.getLocalized()) {
+        distributedCacheManager.releaseCache(c.uri, taskConf, c.timestamp);
+      }
     }
   }
 

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java?rev=885924&r1=885923&r2=885924&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapreduce/filecache/TrackerDistributedCacheManager.java
Tue Dec  1 20:51:36 2009
@@ -131,39 +131,51 @@
       lcacheStatus.refcount++;
     }
     
-    // do the localization, after releasing the global lock
-    synchronized (lcacheStatus) {
-      if (!lcacheStatus.isInited()) {
-        localizedPath = localizeCache(conf, cache, confFileStamp, lcacheStatus,
-            fileStatus, isArchive);
-        lcacheStatus.initComplete();
-      } else {
-        localizedPath = checkCacheStatusValidity(conf, cache, confFileStamp,
-            lcacheStatus, fileStatus, isArchive);
+    boolean initSuccessful = false;
+    try {
+      // do the localization, after releasing the global lock
+      synchronized (lcacheStatus) {
+        if (!lcacheStatus.isInited()) {
+          localizedPath = localizeCache(conf, cache, confFileStamp,
+              lcacheStatus, fileStatus, isArchive);
+          lcacheStatus.initComplete();
+        } else {
+          localizedPath = checkCacheStatusValidity(conf, cache, confFileStamp,
+              lcacheStatus, fileStatus, isArchive);
+        }
+        createSymlink(conf, cache, lcacheStatus, isArchive, currentWorkDir,
+            honorSymLinkConf);
       }
-      createSymlink(conf, cache, lcacheStatus, isArchive,
-          currentWorkDir, honorSymLinkConf);
-    }
 
-    // try deleting stuff if you can
-    long size = 0;
-    synchronized (lcacheStatus) {
-      synchronized (baseDirSize) {
-        Long get = baseDirSize.get(lcacheStatus.getBaseDir());
-        if ( get != null ) {
-         size = get.longValue();
-        } else {
-          LOG.warn("Cannot find size of baseDir: " + lcacheStatus.getBaseDir());
+      // try deleting stuff if you can
+      long size = 0;
+      synchronized (lcacheStatus) {
+        synchronized (baseDirSize) {
+          Long get = baseDirSize.get(lcacheStatus.getBaseDir());
+          if (get != null) {
+            size = get.longValue();
+          } else {
+            LOG.warn("Cannot find size of baseDir: "
+                + lcacheStatus.getBaseDir());
+          }
+        }
+      }
+      // setting the cache size to a default of 10GB
+      long allowedSize = conf.getLong(TTConfig.TT_LOCAL_CACHE_SIZE,
+          DEFAULT_CACHE_SIZE);
+      if (allowedSize < size) {
+        // try some cache deletions
+        deleteCache(conf);
+      }
+      initSuccessful = true;
+      return localizedPath;
+    } finally {
+      if (!initSuccessful) {
+        synchronized (cachedArchives) {
+          lcacheStatus.refcount--;
         }
       }
     }
-    // setting the cache size to a default of 10GB
-    long allowedSize = conf.getLong(TTConfig.TT_LOCAL_CACHE_SIZE, DEFAULT_CACHE_SIZE);
-    if (allowedSize < size) {
-      // try some cache deletions
-      deleteCache(conf);
-    }
-    return localizedPath;
   }
 
   /**
@@ -190,6 +202,21 @@
     }
   }
 
+  /*
+   * This method is called from unit tests. 
+   */
+  int getReferenceCount(URI cache, Configuration conf, long timeStamp) 
+    throws IOException {
+    String key = getKey(cache, conf, timeStamp);
+    synchronized (cachedArchives) {
+      CacheStatus lcacheStatus = cachedArchives.get(key);
+      if (lcacheStatus == null) {
+        throw new IOException("Cannot find localized cache: " + cache);
+      }
+      return lcacheStatus.refcount;
+    }
+  }
+
   // To delete the caches which have a refcount of zero
 
   private void deleteCache(Configuration conf) throws IOException {
@@ -317,7 +344,7 @@
   
   // the method which actually copies the caches locally and unjars/unzips them
   // and does chmod for the files
-  private Path localizeCache(Configuration conf,
+  Path localizeCache(Configuration conf,
                                     URI cache, long confFileStamp,
                                     CacheStatus cacheStatus,
                                     FileStatus fileStatus,
@@ -456,7 +483,7 @@
     }
   }
 
-  private static class CacheStatus {
+  static class CacheStatus {
     // the local load path of this cache
     Path localLoadPath;
 

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
 /hadoop/core/trunk/src/test/mapred:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred:804974-884916
+/hadoop/mapreduce/trunk/src/test/mapred:804974-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/fs:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/fs:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:807679-884916
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/fs:807679-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/hdfs:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/hdfs:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:807679-884916
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/hdfs:807679-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/io/FileBench.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/FileBench.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/FileBench.java:817878-835934,884917-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java:817878-835934,884917-885774

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs-with-mr/org/apache/hadoop/ipc:713112
 /hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/ipc:776175-784663
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs-with-mr/org/apache/hadoop/ipc:796829-820463
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/ipc:807679-884916
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/ipc:807679-885774

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java?rev=885924&r1=885923&r2=885924&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerDistributedCacheManagerWithLinuxTaskController.java
Tue Dec  1 20:51:36 2009
@@ -21,8 +21,6 @@
 import java.io.File;
 import java.io.IOException;
 
-import javax.security.auth.login.LoginException;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -87,19 +85,9 @@
     super.tearDown();
   }
 
-  /**
-   * Test the control flow of distributed cache manager when LinuxTaskController
-   * is used.
-   */
   @Override
-  public void testManagerFlow()
-      throws IOException,
-      LoginException {
-    if (!ClusterWithLinuxTaskController.shouldRun()) {
-      return;
-    }
-
-    super.testManagerFlow();
+  protected boolean canRun() {
+    return ClusterWithLinuxTaskController.shouldRun();
   }
 
   @Override
@@ -165,30 +153,4 @@
       path = path.getParentFile();
     }
   }
-
-  @Override
-  public void testDeleteCache()
-      throws Exception {
-    if (!ClusterWithLinuxTaskController.shouldRun()) {
-      return;
-    }
-    super.testDeleteCache();
-  }
-
-  @Override
-  public void testFileSystemOtherThanDefault()
-      throws Exception {
-    if (!ClusterWithLinuxTaskController.shouldRun()) {
-      return;
-    }
-    super.testFileSystemOtherThanDefault();
-  }
-  
-  @Override
-  public void testFreshness()  throws Exception { 
-    if (!ClusterWithLinuxTaskController.shouldRun()) {
-      return;
-    }
-    super.testFreshness();
-  }
 }

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java?rev=885924&r1=885923&r2=885924&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
Tue Dec  1 20:51:36 2009
@@ -22,6 +22,8 @@
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.Random;
 
 import javax.security.auth.login.LoginException;
@@ -34,6 +36,8 @@
 import org.apache.hadoop.mapred.TaskController;
 import org.apache.hadoop.mapred.TaskTracker;
 import org.apache.hadoop.mapred.TaskController.InitializationContext;
+import org.apache.hadoop.mapreduce.Cluster;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -45,6 +49,7 @@
 import org.apache.hadoop.mapreduce.filecache.TrackerDistributedCacheManager;
 import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.mortbay.log.Log;
 
 public class TestTrackerDistributedCacheManager extends TestCase {
 
@@ -94,12 +99,24 @@
   }
 
   /**
+   * Whether the test can run on the machine
+   * 
+   * @return true if test can run on the machine, false otherwise
+   */
+  protected boolean canRun() {
+    return true;
+  }
+  
+  /**
    * This is the typical flow for using the DistributedCache classes.
    * 
    * @throws IOException
    * @throws LoginException
    */
   public void testManagerFlow() throws IOException, LoginException {
+    if (!canRun()) {
+      return;
+    }
 
     // ****** Imitate JobClient code
     // Configures a task/job with both a regular file and a "classpath" file.
@@ -153,6 +170,99 @@
   }
 
   /**
+   * This DistributedCacheManager fails in localizing firstCacheFile.
+   */
+  public class FakeTrackerDistributedCacheManager extends
+      TrackerDistributedCacheManager {
+    public FakeTrackerDistributedCacheManager(Configuration conf)
+        throws IOException {
+      super(conf);
+    }
+
+    @Override
+    Path localizeCache(Configuration conf, URI cache, long confFileStamp,
+        CacheStatus cacheStatus, FileStatus fileStatus, boolean isArchive)
+        throws IOException {
+      if (cache.equals(firstCacheFile.toUri())) {
+        throw new IOException("fake fail");
+      }
+      return super.localizeCache(conf, cache, confFileStamp, cacheStatus,
+          fileStatus, isArchive);
+    }
+  }
+
+  public void testReferenceCount() throws IOException, LoginException,
+      URISyntaxException {
+    if (!canRun()) {
+      return;
+    }
+    Configuration conf = new Configuration();
+    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
+    TrackerDistributedCacheManager manager = 
+      new FakeTrackerDistributedCacheManager(conf);
+    Cluster cluster = new Cluster(conf);
+    String userName = getJobOwnerName();
+    File workDir = new File(new Path(TEST_ROOT_DIR, "workdir").toString());
+
+    // Configures a job with a regular file
+    Job job1 = Job.getInstance(cluster, conf);
+    job1.addCacheFile(secondCacheFile.toUri());
+    Configuration conf1 = job1.getConfiguration();
+    TrackerDistributedCacheManager.determineTimestamps(conf1);
+
+    // Task localizing for first job
+    TaskDistributedCacheManager handle = manager
+        .newTaskDistributedCacheManager(conf1);
+    handle.setup(localDirAllocator, workDir, TaskTracker
+          .getDistributedCacheDir(userName));
+    handle.release();
+    for (TaskDistributedCacheManager.CacheFile c : handle.getCacheFiles()) {
+      assertEquals(0, manager.getReferenceCount(c.uri, conf1, c.timestamp));
+    }
+    
+    Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile");
+    createTempFile(thirdCacheFile);
+    
+    // Configures another job with three regular files.
+    Job job2 = Job.getInstance(cluster, conf);
+    // add a file that would get failed to localize
+    job2.addCacheFile(firstCacheFile.toUri());
+    // add a file that is already localized by different job
+    job2.addCacheFile(secondCacheFile.toUri());
+    // add a file that is never localized
+    job2.addCacheFile(thirdCacheFile.toUri());
+    Configuration conf2 = job2.getConfiguration();
+    TrackerDistributedCacheManager.determineTimestamps(conf2);
+
+    // Task localizing for second job
+    // localization for the "firstCacheFile" will fail.
+    handle = manager.newTaskDistributedCacheManager(conf2);
+    Throwable th = null;
+    try {
+      handle.setup(localDirAllocator, workDir, TaskTracker
+          .getDistributedCacheDir(userName));
+    } catch (IOException e) {
+      th = e;
+      Log.info("Exception during setup", e);
+    }
+    assertNotNull(th);
+    assertTrue(th.getMessage().contains("fake fail"));
+    handle.release();
+    th = null;
+    for (TaskDistributedCacheManager.CacheFile c : handle.getCacheFiles()) {
+      try {
+        assertEquals(0, manager.getReferenceCount(c.uri, conf2, c.timestamp));
+      } catch (IOException ie) {
+        th = ie;
+        Log.info("Exception getting reference count for " + c.uri, ie);
+      }
+    }
+    assertNotNull(th);
+    assertTrue(th.getMessage().contains(thirdCacheFile.getName()));
+    fs.delete(thirdCacheFile, false);
+  }
+
+  /**
    * Check proper permissions on the cache files
    * 
    * @param localCacheFiles
@@ -180,6 +290,9 @@
 
   /** test delete cache */
   public void testDeleteCache() throws Exception {
+    if (!canRun()) {
+      return;
+    }
     TrackerDistributedCacheManager manager = 
         new TrackerDistributedCacheManager(conf);
     FileSystem localfs = FileSystem.getLocal(conf);
@@ -204,6 +317,9 @@
   }
   
   public void testFileSystemOtherThanDefault() throws Exception {
+    if (!canRun()) {
+      return;
+    }
     TrackerDistributedCacheManager manager =
       new TrackerDistributedCacheManager(conf);
     conf.set("fs.fakefile.impl", conf.get("fs.file.impl"));
@@ -262,6 +378,9 @@
   }
   
   public void testFreshness() throws Exception {
+    if (!canRun()) {
+      return;
+    }
     Configuration myConf = new Configuration(conf);
     myConf.set("fs.default.name", "refresh:///");
     myConf.setClass("fs.refresh.impl", FakeFileSystem.class, FileSystem.class);

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:713112
 /hadoop/core/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934
+/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java:817878-835934,884917-885774

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java?rev=885924&r1=885923&r2=885924&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
Tue Dec  1 20:51:36 2009
@@ -452,6 +452,81 @@
     }
   }
 
+  public void testCopyDfsToDfsUpdateWithSkipCRC() throws Exception {
+    MiniDFSCluster cluster = null;
+    try {
+      Configuration conf = new Configuration();
+      cluster = new MiniDFSCluster(conf, 2, true, null);
+      final FileSystem hdfs = cluster.getFileSystem();
+      final String namenode = hdfs.getUri().toString();
+      
+      FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
+      // Create two files of the same name, same length but different
+      // contents
+      final String testfilename = "test";
+      final String srcData = "act act act";
+      final String destData = "cat cat cat";
+      
+      if (namenode.startsWith("hdfs://")) {
+        deldir(hdfs,"/logs");
+        
+        Path srcPath = new Path("/srcdat", testfilename);
+        Path destPath = new Path("/destdat", testfilename);
+        FSDataOutputStream out = fs.create(srcPath, true);
+        out.writeUTF(srcData);
+        out.close();
+
+        out = fs.create(destPath, true);
+        out.writeUTF(destData);
+        out.close();
+        
+        // Run with -skipcrccheck option
+        ToolRunner.run(new DistCp(conf), new String[] {
+          "-p",
+          "-update",
+          "-skipcrccheck",
+          "-log",
+          namenode+"/logs",
+          namenode+"/srcdat",
+          namenode+"/destdat"});
+        
+        // File should not be overwritten
+        FSDataInputStream in = hdfs.open(destPath);
+        String s = in.readUTF();
+        System.out.println("Dest had: " + s);
+        assertTrue("Dest got over written even with skip crc",
+            s.equalsIgnoreCase(destData));
+        in.close();
+        
+        deldir(hdfs, "/logs");
+
+        // Run without the option        
+        ToolRunner.run(new DistCp(conf), new String[] {
+          "-p",
+          "-update",
+          "-log",
+          namenode+"/logs",
+          namenode+"/srcdat",
+          namenode+"/destdat"});
+        
+        // File should be overwritten
+        in = hdfs.open(destPath);
+        s = in.readUTF();
+        System.out.println("Dest had: " + s);
+
+        assertTrue("Dest did not get overwritten without skip crc",
+            s.equalsIgnoreCase(srcData));
+        in.close();
+
+        deldir(hdfs, "/destdat");
+        deldir(hdfs, "/srcdat");
+        deldir(hdfs, "/logs");
+       }
+    } finally {
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+    
   public void testCopyDuplication() throws Exception {
     final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
     try {    

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/tools/org/apache/hadoop/tools/DistCp.java?rev=885924&r1=885923&r2=885924&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/tools/org/apache/hadoop/tools/DistCp.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/tools/org/apache/hadoop/tools/DistCp.java
Tue Dec  1 20:51:36 2009
@@ -97,6 +97,9 @@
     "\n-m <num_maps>          Maximum number of simultaneous copies" +
     "\n-overwrite             Overwrite destination" +
     "\n-update                Overwrite if src size different from dst size" +
+    "\n-skipcrccheck          Do not use CRC check to determine if src is " +
+    "\n                       different from dest. Relevant only if -update" +
+    "\n                       is specified" +
     "\n-f <urilist_uri>       Use list at <urilist_uri> as src list" +
     "\n-filelimit <n>         Limit the total number of files to be <= n" +
     "\n-sizelimit <n>         Limit the total size to be <= n bytes" +
@@ -134,7 +137,8 @@
     IGNORE_READ_FAILURES("-i", NAME + ".ignore.read.failures"),
     PRESERVE_STATUS("-p", NAME + ".preserve.status"),
     OVERWRITE("-overwrite", NAME + ".overwrite.always"),
-    UPDATE("-update", NAME + ".overwrite.ifnewer");
+    UPDATE("-update", NAME + ".overwrite.ifnewer"),
+    SKIPCRC("-skipcrccheck", NAME + ".skip.crc.check");
 
     final String cmd, propertyname;
 
@@ -327,7 +331,8 @@
     private Path destPath = null;
     private byte[] buffer = null;
     private JobConf job;
-
+    private boolean skipCRCCheck = false;
+    
     // stats
     private int failcount = 0;
     private int skipcount = 0;
@@ -351,7 +356,7 @@
     private boolean needsUpdate(FileStatus srcstatus,
         FileSystem dstfs, Path dstpath) throws IOException {
       return update && !sameFile(srcstatus.getPath().getFileSystem(job),
-          srcstatus, dstfs, dstpath);
+          srcstatus, dstfs, dstpath, skipCRCCheck);
     }
     
     private FSDataOutputStream create(Path f, Reporter reporter,
@@ -384,7 +389,7 @@
             throws IOException {
       if (destFileSys.exists(absdst)) {
         if (sameFile(srcstat.getPath().getFileSystem(job), srcstat,
-            destFileSys, absdst)) {
+            destFileSys, absdst, skipCRCCheck)) {
           return true;
         }
       }
@@ -622,6 +627,7 @@
       }
       update = job.getBoolean(Options.UPDATE.propertyname, false);
       overwrite = !update && job.getBoolean(Options.OVERWRITE.propertyname, false);
+      skipCRCCheck = job.getBoolean(Options.SKIPCRC.propertyname, false);
       this.job = job;
     }
 
@@ -965,9 +971,15 @@
       final boolean isOverwrite = flags.contains(Options.OVERWRITE);
       final boolean isUpdate = flags.contains(Options.UPDATE);
       final boolean isDelete = flags.contains(Options.DELETE);
+      final boolean skipCRC = flags.contains(Options.SKIPCRC);
       if (isOverwrite && isUpdate) {
         throw new IllegalArgumentException("Conflicting overwrite policies");
       }
+      if (!isUpdate && skipCRC) {
+        throw new IllegalArgumentException(
+            Options.SKIPCRC.cmd + " is relevant only with the " +
+            Options.UPDATE.cmd + " option");
+      }
       if (isDelete && !isOverwrite && !isUpdate) {
         throw new IllegalArgumentException(Options.DELETE.cmd
             + " must be specified with " + Options.OVERWRITE + " or "
@@ -1171,9 +1183,11 @@
 
     //set boolean values
     final boolean update = args.flags.contains(Options.UPDATE);
+    final boolean skipCRCCheck = args.flags.contains(Options.SKIPCRC);
     final boolean overwrite = !update && args.flags.contains(Options.OVERWRITE)
                               && !args.dryrun;
     jobConf.setBoolean(Options.UPDATE.propertyname, update);
+    jobConf.setBoolean(Options.SKIPCRC.propertyname, skipCRCCheck);
     jobConf.setBoolean(Options.OVERWRITE.propertyname, overwrite);
     jobConf.setBoolean(Options.IGNORE_READ_FAILURES.propertyname,
         args.flags.contains(Options.IGNORE_READ_FAILURES));
@@ -1335,7 +1349,8 @@
                 }
               }
               //skip path if the src and the dst files are the same.
-              skipPath = update && sameFile(srcfs, child, dstfs, destPath);
+              skipPath = update && 
+              	sameFile(srcfs, child, dstfs, destPath, skipCRCCheck);
               //skip path if it exceed file limit or size limit
               skipPath |= fileCount == args.filelimit
                           || byteCount + child.getLen() > args.sizelimit; 
@@ -1460,7 +1475,7 @@
    * two files are considered as the same if they have the same size.
    */
   static private boolean sameFile(FileSystem srcfs, FileStatus srcstatus,
-      FileSystem dstfs, Path dstpath) throws IOException {
+      FileSystem dstfs, Path dstpath, boolean skipCRCCheck) throws IOException {
     FileStatus dststatus;
     try {
       dststatus = dstfs.getFileStatus(dstpath);
@@ -1473,6 +1488,11 @@
       return false;
     }
 
+    if (skipCRCCheck) {
+      LOG.debug("Skipping the CRC check");
+      return true;
+    }
+    
     //get src checksum
     final FileChecksum srccs;
     try {

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Dec  1 20:51:36 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
 /hadoop/core/trunk/src/webapps/job:776175-785643
-/hadoop/mapreduce/trunk/src/webapps/job:804974-884916
+/hadoop/mapreduce/trunk/src/webapps/job:804974-885774



Mime
View raw message