hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From inigo...@apache.org
Subject [02/50] [abbrv] hadoop git commit: HADOOP-9631. ViewFs should use underlying FileSystem's server side defaults. Contributed by Lohit Vijayarenu and Erik Krogen.
Date Tue, 28 Mar 2017 21:25:25 GMT
HADOOP-9631. ViewFs should use underlying FileSystem's server side defaults. Contributed by
Lohit Vijayarenu and Erik Krogen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59d69257
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59d69257
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59d69257

Branch: refs/heads/HDFS-10467
Commit: 59d69257a888347f0fb9c51bb000afc986b64f98
Parents: ed2d03f
Author: Zhe Zhang <zhz@apache.org>
Authored: Wed Mar 22 22:21:30 2017 -0700
Committer: Zhe Zhang <zhz@apache.org>
Committed: Wed Mar 22 22:22:22 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/fs/AbstractFileSystem.java    | 16 ++++++-
 .../java/org/apache/hadoop/fs/ChecksumFs.java   | 13 +++---
 .../apache/hadoop/fs/DelegateToFileSystem.java  |  6 +++
 .../java/org/apache/hadoop/fs/FilterFs.java     |  5 ++
 .../java/org/apache/hadoop/fs/ftp/FtpFs.java    |  7 +++
 .../org/apache/hadoop/fs/local/RawLocalFs.java  |  8 ++++
 .../org/apache/hadoop/fs/viewfs/ChRootedFs.java |  6 +++
 .../org/apache/hadoop/fs/viewfs/ViewFs.java     | 20 +++++++-
 .../org/apache/hadoop/fs/TestAfsCheckPath.java  |  1 +
 .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 49 ++++++++++++++++++++
 .../main/java/org/apache/hadoop/fs/Hdfs.java    |  6 +++
 11 files changed, 128 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index d1e7a1b..ef68437 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -450,10 +450,22 @@ public abstract class AbstractFileSystem {
    * @return server default configuration values
    * 
    * @throws IOException an I/O error occurred
+   * @deprecated use {@link #getServerDefaults(Path)} instead
    */
+  @Deprecated
   public abstract FsServerDefaults getServerDefaults() throws IOException; 
 
   /**
+   * Return a set of server default configuration values based on path.
+   * @param f path to fetch server defaults
+   * @return server default configuration values for path
+   * @throws IOException an I/O error occurred
+   */
+  public FsServerDefaults getServerDefaults(final Path f) throws IOException {
+    return getServerDefaults();
+  }
+
+  /**
    * Return the fully-qualified path of path f resolving the path
    * through any internal symlinks or mount point
    * @param p path to be resolved
@@ -548,7 +560,7 @@ public abstract class AbstractFileSystem {
     }
 
 
-    FsServerDefaults ssDef = getServerDefaults();
+    FsServerDefaults ssDef = getServerDefaults(f);
     if (ssDef.getBlockSize() % ssDef.getBytesPerChecksum() != 0) {
       throw new IOException("Internal error: default blockSize is" + 
           " not a multiple of default bytesPerChecksum ");
@@ -626,7 +638,7 @@ public abstract class AbstractFileSystem {
    */
   public FSDataInputStream open(final Path f) throws AccessControlException,
       FileNotFoundException, UnresolvedLinkException, IOException {
-    return open(f, getServerDefaults().getFileBufferSize());
+    return open(f, getServerDefaults(f).getFileBufferSize());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
index 3972033..0a8cc73 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
@@ -57,7 +57,7 @@ public abstract class ChecksumFs extends FilterFs {
     throws IOException, URISyntaxException {
     super(theFs);
     defaultBytesPerChecksum = 
-      getMyFs().getServerDefaults().getBytesPerChecksum();
+      getMyFs().getServerDefaults(new Path("/")).getBytesPerChecksum();
   }
   
   /**
@@ -96,9 +96,10 @@ public abstract class ChecksumFs extends FilterFs {
     return defaultBytesPerChecksum;
   }
 
-  private int getSumBufferSize(int bytesPerSum, int bufferSize)
+  private int getSumBufferSize(int bytesPerSum, int bufferSize, Path file)
     throws IOException {
-    int defaultBufferSize =  getMyFs().getServerDefaults().getFileBufferSize();
+    int defaultBufferSize = getMyFs().getServerDefaults(file)
+        .getFileBufferSize();
     int proportionalBufferSize = bufferSize / bytesPerSum;
     return Math.max(bytesPerSum,
                     Math.max(proportionalBufferSize, defaultBufferSize));
@@ -121,7 +122,7 @@ public abstract class ChecksumFs extends FilterFs {
     
     public ChecksumFSInputChecker(ChecksumFs fs, Path file)
       throws IOException, UnresolvedLinkException {
-      this(fs, file, fs.getServerDefaults().getFileBufferSize());
+      this(fs, file, fs.getServerDefaults(file).getFileBufferSize());
     }
     
     public ChecksumFSInputChecker(ChecksumFs fs, Path file, int bufferSize)
@@ -132,7 +133,7 @@ public abstract class ChecksumFs extends FilterFs {
       Path sumFile = fs.getChecksumFile(file);
       try {
         int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(),
-                                                bufferSize);
+            bufferSize, file);
         sums = fs.getRawFs().open(sumFile, sumBufferSize);
 
         byte[] version = new byte[CHECKSUM_VERSION.length];
@@ -353,7 +354,7 @@ public abstract class ChecksumFs extends FilterFs {
       
       // Now create the chekcsumfile; adjust the buffsize
       int bytesPerSum = fs.getBytesPerSum();
-      int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
+      int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize, file);
       this.sums = fs.getRawFs().createInternal(fs.getChecksumFile(file),
           EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
           absolutePermission, sumBufferSize, replication, blockSize, progress,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
index d2550dc..a5ab75e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
@@ -149,11 +149,17 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem
{
   }
 
   @Override
+  @Deprecated
   public FsServerDefaults getServerDefaults() throws IOException {
     return fsImpl.getServerDefaults();
   }
   
   @Override
+  public FsServerDefaults getServerDefaults(final Path f) throws IOException {
+    return fsImpl.getServerDefaults(f);
+  }
+
+  @Override
   public Path getHomeDirectory() {
     return fsImpl.getHomeDirectory();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index d7e313f..6b1093e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -147,10 +147,15 @@ public abstract class FilterFs extends AbstractFileSystem {
   }
 
   @Override
+  @Deprecated
   public FsServerDefaults getServerDefaults() throws IOException {
     return myFs.getServerDefaults();
   }
   
+  @Override
+  public FsServerDefaults getServerDefaults(final Path f) throws IOException {
+    return myFs.getServerDefaults(f);
+  }
 
   @Override
   public Path resolvePath(final Path p) throws FileNotFoundException,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java
index 0a97375..6d54e36 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.DelegateToFileSystem;
 import org.apache.hadoop.fs.FsConstants;
 import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.Path;
 
 /**
  * The FtpFs implementation of AbstractFileSystem.
@@ -57,7 +58,13 @@ public class FtpFs extends DelegateToFileSystem {
   }
   
   @Override
+  @Deprecated
   public FsServerDefaults getServerDefaults() throws IOException {
     return FtpConfigKeys.getServerDefaults();
   }
+
+  @Override
+  public FsServerDefaults getServerDefaults(final Path f) throws IOException {
+    return FtpConfigKeys.getServerDefaults();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
index 6cb2792..8aebe02 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.DelegateToFileSystem;
 import org.apache.hadoop.fs.FsConstants;
 import org.apache.hadoop.fs.FsServerDefaults;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RawLocalFileSystem;
 
 /**
@@ -63,6 +64,13 @@ public class RawLocalFs extends DelegateToFileSystem {
   }
   
   @Override
+  public FsServerDefaults getServerDefaults(final Path f)
+      throws IOException {
+    return LocalConfigKeys.getServerDefaults();
+  }
+
+  @Override
+  @Deprecated
   public FsServerDefaults getServerDefaults() throws IOException {
     return LocalConfigKeys.getServerDefaults();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index b7e47c2..d77ad8b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -223,11 +223,17 @@ class ChRootedFs extends AbstractFileSystem {
   }
 
   @Override
+  @Deprecated
   public FsServerDefaults getServerDefaults() throws IOException {
     return myFs.getServerDefaults();
   }
 
   @Override
+  public FsServerDefaults getServerDefaults(final Path f) throws IOException {
+    return myFs.getServerDefaults(fullPath(f));
+  }
+
+  @Override
   public int getUriDefaultPort() {
     return myFs.getUriDefaultPort();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 1a54a81..3a34a91 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -240,11 +240,23 @@ public class ViewFs extends AbstractFileSystem {
   }
 
   @Override
+  @Deprecated
   public FsServerDefaults getServerDefaults() throws IOException {
     return LocalConfigKeys.getServerDefaults(); 
   }
 
   @Override
+  public FsServerDefaults getServerDefaults(final Path f) throws IOException {
+    InodeTree.ResolveResult<AbstractFileSystem> res;
+    try {
+      res = fsState.resolve(getUriPath(f), true);
+    } catch (FileNotFoundException fnfe) {
+      return LocalConfigKeys.getServerDefaults();
+    }
+    return res.targetFileSystem.getServerDefaults(res.remainingPath);
+  }
+
+  @Override
   public int getUriDefaultPort() {
     return -1;
   }
@@ -927,8 +939,14 @@ public class ViewFs extends AbstractFileSystem {
     }
 
     @Override
+    @Deprecated
     public FsServerDefaults getServerDefaults() throws IOException {
-      throw new IOException("FsServerDefaults not implemented yet");
+      return LocalConfigKeys.getServerDefaults();
+    }
+
+    @Override
+    public FsServerDefaults getServerDefaults(final Path f) throws IOException {
+      return LocalConfigKeys.getServerDefaults();
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
index 3bd14f1..da429ff 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
@@ -117,6 +117,7 @@ public class TestAfsCheckPath {
     }
 
     @Override
+    @Deprecated
     public FsServerDefaults getServerDefaults() throws IOException {
       // deliberately empty
       return null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
index 8f6df32..50237d1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
@@ -39,6 +39,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -46,8 +47,10 @@ import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileContextTestHelper;
+import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.FileContextTestHelper.fileType;
@@ -55,6 +58,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FsConstants;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.local.LocalConfigKeys;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.AclUtil;
@@ -839,6 +843,51 @@ abstract public class ViewFsBaseTest {
     });
   }
 
+  @Test
+  public void testRespectsServerDefaults() throws Exception {
+    FsServerDefaults targetDefs =
+        fcTarget.getDefaultFileSystem().getServerDefaults(new Path("/"));
+    FsServerDefaults viewDefs =
+        fcView.getDefaultFileSystem().getServerDefaults(new Path("/data"));
+    assertEquals(targetDefs.getReplication(), viewDefs.getReplication());
+    assertEquals(targetDefs.getBlockSize(), viewDefs.getBlockSize());
+    assertEquals(targetDefs.getBytesPerChecksum(),
+        viewDefs.getBytesPerChecksum());
+    assertEquals(targetDefs.getFileBufferSize(),
+        viewDefs.getFileBufferSize());
+    assertEquals(targetDefs.getWritePacketSize(),
+        viewDefs.getWritePacketSize());
+    assertEquals(targetDefs.getEncryptDataTransfer(),
+        viewDefs.getEncryptDataTransfer());
+    assertEquals(targetDefs.getTrashInterval(), viewDefs.getTrashInterval());
+    assertEquals(targetDefs.getChecksumType(), viewDefs.getChecksumType());
+
+    fcView.create(new Path("/data/file"), EnumSet.of(CreateFlag.CREATE))
+        .close();
+    FileStatus stat =
+        fcTarget.getFileStatus(new Path(targetTestRoot, "data/file"));
+    assertEquals(targetDefs.getReplication(), stat.getReplication());
+  }
+
+  @Test
+  public void testServerDefaultsInternalDir() throws Exception {
+    FsServerDefaults localDefs = LocalConfigKeys.getServerDefaults();
+    FsServerDefaults viewDefs = fcView
+        .getDefaultFileSystem().getServerDefaults(new Path("/internalDir"));
+    assertEquals(localDefs.getReplication(), viewDefs.getReplication());
+    assertEquals(localDefs.getBlockSize(), viewDefs.getBlockSize());
+    assertEquals(localDefs.getBytesPerChecksum(),
+        viewDefs.getBytesPerChecksum());
+    assertEquals(localDefs.getFileBufferSize(),
+        viewDefs.getFileBufferSize());
+    assertEquals(localDefs.getWritePacketSize(),
+        viewDefs.getWritePacketSize());
+    assertEquals(localDefs.getEncryptDataTransfer(),
+        viewDefs.getEncryptDataTransfer());
+    assertEquals(localDefs.getTrashInterval(), viewDefs.getTrashInterval());
+    assertEquals(localDefs.getChecksumType(), viewDefs.getChecksumType());
+  }
+
   // Confirm that listLocatedStatus is delegated properly to the underlying
   // AbstractFileSystem to allow for optimizations
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59d69257/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 82ee41a..645f1ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -155,11 +155,17 @@ public class Hdfs extends AbstractFileSystem {
   }
 
   @Override
+  @Deprecated
   public FsServerDefaults getServerDefaults() throws IOException {
     return dfs.getServerDefaults();
   }
 
   @Override
+  public FsServerDefaults getServerDefaults(final Path f) throws IOException {
+    return dfs.getServerDefaults();
+  }
+
+  @Override
   public RemoteIterator<LocatedFileStatus> listLocatedStatus(
       final Path p)
       throws FileNotFoundException, IOException {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message