hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject svn commit: r1331441 - in /hadoop/common/branches/branch-2/hadoop-tools: hadoop-archives/src/main/java/org/apache/hadoop/tools/ hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/ hadoop-distcp/src/test/java/org/apache/hadoop/tools/ hadoop-dist...
Date Fri, 27 Apr 2012 14:41:39 GMT
Author: bobby
Date: Fri Apr 27 14:41:38 2012
New Revision: 1331441

URL: http://svn.apache.org/viewvc?rev=1331441&view=rev
Log:
svn merge -c 1331440. FIXES: HADOOP-8305. distcp over viewfs is broken (John George via bobby)

Added:
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
      - copied unchanged from r1331440, hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
Modified:
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
    hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java?rev=1331441&r1=1331440&r2=1331441&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
Fri Apr 27 14:41:38 2012
@@ -613,7 +613,7 @@ public class HadoopArchives implements T
           destFs.delete(tmpOutput, false);
         } 
         partStream = destFs.create(tmpOutput, false, conf.getInt("io.file.buffer.size", 4096),

-            destFs.getDefaultReplication(), blockSize);
+            destFs.getDefaultReplication(tmpOutput), blockSize);
       } catch(IOException ie) {
         throw new RuntimeException("Unable to open output file " + tmpOutput, ie);
       }

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java?rev=1331441&r1=1331440&r2=1331441&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
Fri Apr 27 14:41:38 2012
@@ -107,8 +107,8 @@ public class RetriableFileCopyCommand ex
                              throws IOException {
     OutputStream outStream = new BufferedOutputStream(targetFS.create(
             tmpTargetPath, true, BUFFER_SIZE,
-            getReplicationFactor(fileAttributes, sourceFileStatus, targetFS),
-            getBlockSize(fileAttributes, sourceFileStatus, targetFS), context));
+            getReplicationFactor(fileAttributes, sourceFileStatus, targetFS, tmpTargetPath),
+            getBlockSize(fileAttributes, sourceFileStatus, targetFS, tmpTargetPath), context));
     return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, true, context);
   }
 
@@ -218,16 +218,16 @@ public class RetriableFileCopyCommand ex
 
   private static short getReplicationFactor(
           EnumSet<FileAttribute> fileAttributes,
-          FileStatus sourceFile, FileSystem targetFS) {
+          FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
     return fileAttributes.contains(FileAttribute.REPLICATION)?
-            sourceFile.getReplication() : targetFS.getDefaultReplication();
+            sourceFile.getReplication() : targetFS.getDefaultReplication(tmpTargetPath);
   }
 
   private static long getBlockSize(
           EnumSet<FileAttribute> fileAttributes,
-          FileStatus sourceFile, FileSystem targetFS) {
+          FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
     return fileAttributes.contains(FileAttribute.BLOCKSIZE)?
-            sourceFile.getBlockSize() : targetFS.getDefaultBlockSize();
+            sourceFile.getBlockSize() : targetFS.getDefaultBlockSize(tmpTargetPath);
   }
 
   /**

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java?rev=1331441&r1=1331440&r2=1331441&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
Fri Apr 27 14:41:38 2012
@@ -110,9 +110,9 @@ public class TestDistCp {
       fs = cluster.getFileSystem();
       final Path qualifiedPath = new Path(path).makeQualified(fs.getUri(),
                                             fs.getWorkingDirectory());
-      final long blockSize = fs.getDefaultBlockSize() * 2;
+      final long blockSize = fs.getDefaultBlockSize(new Path(path)) * 2;
       outputStream = fs.create(qualifiedPath, true, 0,
-              (short)(fs.getDefaultReplication()*2),
+              (short)(fs.getDefaultReplication(new Path(path))*2),
               blockSize);
       outputStream.write(new byte[FILE_SIZE]);
       pathList.add(qualifiedPath);

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java?rev=1331441&r1=1331440&r2=1331441&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
Fri Apr 27 14:41:38 2012
@@ -127,9 +127,9 @@ public class TestCopyMapper {
       fs = cluster.getFileSystem();
       final Path qualifiedPath = new Path(path).makeQualified(fs.getUri(),
                                                       fs.getWorkingDirectory());
-      final long blockSize = fs.getDefaultBlockSize() * 2;
+      final long blockSize = fs.getDefaultBlockSize(qualifiedPath) * 2;
       outputStream = fs.create(qualifiedPath, true, 0,
-              (short)(fs.getDefaultReplication()*2),
+              (short)(fs.getDefaultReplication(qualifiedPath)*2),
               blockSize);
       outputStream.write(new byte[FILE_SIZE]);
       pathList.add(qualifiedPath);

Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java?rev=1331441&r1=1331440&r2=1331441&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
Fri Apr 27 14:41:38 2012
@@ -374,9 +374,9 @@ public class DistCp implements Tool {
       FsPermission permission = preseved.contains(FileAttribute.PERMISSION)?
           srcstat.getPermission(): null;
       short replication = preseved.contains(FileAttribute.REPLICATION)?
-          srcstat.getReplication(): destFileSys.getDefaultReplication();
+          srcstat.getReplication(): destFileSys.getDefaultReplication(f);
       long blockSize = preseved.contains(FileAttribute.BLOCK_SIZE)?
-          srcstat.getBlockSize(): destFileSys.getDefaultBlockSize();
+          srcstat.getBlockSize(): destFileSys.getDefaultBlockSize(f);
       return destFileSys.create(f, permission, true, sizeBuf, replication,
           blockSize, reporter);
     }



Mime
View raw message