hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From y...@apache.org
Subject hadoop git commit: HADOOP-11510. Expose truncate API via FileContext. (yliu)
Date Tue, 10 Feb 2015 02:42:04 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 4d4442cb3 -> ae316705b


HADOOP-11510. Expose truncate API via FileContext. (yliu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae316705
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae316705
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae316705

Branch: refs/heads/branch-2
Commit: ae316705bb79479038a13b80bab6febbe8f3c75f
Parents: 4d4442c
Author: yliu <yliu@apache.org>
Authored: Tue Feb 10 01:43:08 2015 +0800
Committer: yliu <yliu@apache.org>
Committed: Tue Feb 10 01:43:08 2015 +0800

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  2 +
 .../apache/hadoop/fs/AbstractFileSystem.java    |  9 ++++
 .../java/org/apache/hadoop/fs/ChecksumFs.java   |  5 +++
 .../apache/hadoop/fs/DelegateToFileSystem.java  |  6 +++
 .../java/org/apache/hadoop/fs/FileContext.java  | 43 ++++++++++++++++++++
 .../java/org/apache/hadoop/fs/FilterFs.java     |  8 ++++
 .../org/apache/hadoop/fs/viewfs/ChRootedFs.java |  6 +++
 .../org/apache/hadoop/fs/viewfs/ViewFs.java     | 17 +++++++-
 .../org/apache/hadoop/fs/TestAfsCheckPath.java  |  6 +++
 .../main/java/org/apache/hadoop/fs/Hdfs.java    |  6 +++
 .../fs/TestHDFSFileContextMainOperations.java   | 32 ++++++++++++++-
 11 files changed, 138 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8d1047f..a61c349 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -31,6 +31,8 @@ Release 2.7.0 - UNRELEASED
     HADOOP-11045. Introducing a tool to detect flaky tests of hadoop jenkins testing
     job. (Yongjun Zhang and Todd Lipcon via ozawa)
 
+    HADOOP-11510. Expose truncate API via FileContext. (yliu)
+
   IMPROVEMENTS
 
     HADOOP-11483. HardLink.java should use the jdk7 createLink method (aajisaka)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index 92d4eca..975cc3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -639,6 +639,15 @@ public abstract class AbstractFileSystem {
 
   /**
    * The specification of this method matches that of
+   * {@link FileContext#truncate(Path, long)} except that Path f must be for
+   * this file system.
+   */
+  public abstract boolean truncate(Path f, long newLength)
+      throws AccessControlException, FileNotFoundException,
+      UnresolvedLinkException, IOException;
+
+  /**
+   * The specification of this method matches that of
    * {@link FileContext#setReplication(Path, short)} except that Path f must be
    * for this file system.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
index ab5cd13..7dc4a80 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
@@ -297,6 +297,11 @@ public abstract class ChecksumFs extends FilterFs {
 
   }
 
+  @Override
+  public boolean truncate(Path f, long newLength) throws IOException {
+    throw new IOException("Not supported");
+  }
+
   /**
    * Opens an FSDataInputStream at the indicated Path.
    * @param f the file name to open

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
index 1cdcb27..09707c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
@@ -170,6 +170,12 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem
{
   }
 
   @Override
+  public boolean truncate(Path f, long newLength) throws IOException {
+    checkPath(f);
+    return fsImpl.truncate(f, newLength);
+  }
+
+  @Override
   @SuppressWarnings("deprecation") // call to rename
   public void renameInternal(Path src, Path dst) throws IOException {
     checkPath(src);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 3c5e9ab..705a8d8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -829,6 +829,49 @@ public class FileContext {
   }
 
   /**
+   * Truncate the file in the indicated path to the indicated size.
+   * <ul>
+   * <li>Fails if path is a directory.
+   * <li>Fails if path does not exist.
+   * <li>Fails if path is not closed.
+   * <li>Fails if new size is greater than current size.
+   * </ul>
+   * @param f The path to the file to be truncated
+   * @param newLength The size the file is to be truncated to
+   *
+   * @return <code>true</code> if the file has been truncated to the desired
+   * <code>newLength</code> and is immediately available to be reused for
+   * write operations such as <code>append</code>, or
+   * <code>false</code> if a background process of adjusting the length of
+   * the last block has been started, and clients should wait for it to
+   * complete before proceeding with further file updates.
+   *
+   * @throws AccessControlException If access is denied
+   * @throws FileNotFoundException If file <code>f</code> does not exist
+   * @throws UnsupportedFileSystemException If file system for <code>f</code>
is
+   *           not supported
+   * @throws IOException If an I/O error occurred
+   *
+   * Exceptions applicable to file systems accessed over RPC:
+   * @throws RpcClientException If an exception occurred in the RPC client
+   * @throws RpcServerException If an exception occurred in the RPC server
+   * @throws UnexpectedServerException If server implementation throws
+   *           undeclared exception to RPC server
+   */
+  public boolean truncate(final Path f, final long newLength)
+      throws AccessControlException, FileNotFoundException,
+      UnsupportedFileSystemException, IOException {
+    final Path absF = fixRelativePart(f);
+    return new FSLinkResolver<Boolean>() {
+      @Override
+      public Boolean next(final AbstractFileSystem fs, final Path p)
+          throws IOException, UnresolvedLinkException {
+        return fs.truncate(p, newLength);
+      }
+    }.resolve(this, absF);
+  }
+
+  /**
    * Set replication for an existing file.
    * 
    * @param f file name

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index b6e1d96..4f28a9a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -213,6 +213,14 @@ public abstract class FilterFs extends AbstractFileSystem {
   }
 
   @Override
+  public boolean truncate(Path f, long newLength) 
+      throws AccessControlException, FileNotFoundException,
+      UnresolvedLinkException, IOException {
+    checkPath(f);
+    return myFs.truncate(f, newLength);
+  }
+
+  @Override
   public void renameInternal(Path src, Path dst) 
     throws IOException, UnresolvedLinkException {
     checkPath(src);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index 9569e10..68e756a8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -248,6 +248,12 @@ class ChRootedFs extends AbstractFileSystem {
   }
 
   @Override
+  public boolean truncate(final Path f, final long newLength)
+      throws IOException, UnresolvedLinkException {
+    return myFs.truncate(fullPath(f), newLength);
+  }
+
+  @Override
   public void renameInternal(final Path src, final Path dst)
     throws IOException, UnresolvedLinkException {
     // note fullPath will check that paths are relative to this FileSystem.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 014f488..975496a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -452,7 +452,15 @@ public class ViewFs extends AbstractFileSystem {
     return res.targetFileSystem.open(res.remainingPath, bufferSize);
   }
 
-  
+  @Override
+  public boolean truncate(final Path f, final long newLength)
+      throws AccessControlException, FileNotFoundException,
+      UnresolvedLinkException, IOException {
+    InodeTree.ResolveResult<AbstractFileSystem> res =
+        fsState.resolve(getUriPath(f), true);
+    return res.targetFileSystem.truncate(res.remainingPath, newLength);
+  }
+
   @Override
   public void renameInternal(final Path src, final Path dst,
       final boolean overwrite) throws IOException, UnresolvedLinkException {
@@ -878,6 +886,13 @@ public class ViewFs extends AbstractFileSystem {
     }
 
     @Override
+    public boolean truncate(final Path f, final long newLength)
+        throws FileNotFoundException, IOException {
+      checkPathIsSlash(f);
+      throw readOnlyMountTable("truncate", f);
+    }
+
+    @Override
     public void renameInternal(final Path src, final Path dst)
         throws AccessControlException, IOException {
       checkPathIsSlash(src);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
index 3bd14f1..6b9378d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
@@ -141,6 +141,12 @@ public class TestAfsCheckPath {
     }
 
     @Override
+    public boolean truncate(Path f, long newLength) throws IOException {
+      // deliberately empty
+      return false;
+    }
+
+    @Override
     public void renameInternal(Path src, Path dst) throws IOException {
       // deliberately empty
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 1b9f515..8c09193 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -318,6 +318,12 @@ public class Hdfs extends AbstractFileSystem {
   }
 
   @Override
+  public boolean truncate(Path f, long newLength)
+      throws IOException, UnresolvedLinkException {
+    return dfs.truncate(getUriPath(f), newLength);
+  }
+
+  @Override
   public void renameInternal(Path src, Path dst) 
     throws IOException, UnresolvedLinkException {
     dfs.rename(getUriPath(src), getUriPath(dst), Options.Rename.NONE);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae316705/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
index 8a215b0..94fb0fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java
@@ -28,6 +28,7 @@ import java.net.URISyntaxException;
 import javax.security.auth.login.LoginException;
 
 import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.hdfs.AppendTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -114,7 +115,36 @@ public class TestHDFSFileContextMainOperations extends
   private Path getTestRootPath(FileContext fc, String path) {
     return fileContextTestHelper.getTestRootPath(fc, path);
   }
-  
+
+  @Test
+  public void testTruncate() throws Exception {
+    final short repl = 3;
+    final int blockSize = 1024;
+    final int numOfBlocks = 2;
+    DistributedFileSystem fs = cluster.getFileSystem();
+    Path dir = getTestRootPath(fc, "test/hadoop");
+    Path file = getTestRootPath(fc, "test/hadoop/file");
+
+    final byte[] data = FileSystemTestHelper.getFileData(
+        numOfBlocks, blockSize);
+    FileSystemTestHelper.createFile(fs, file, data, blockSize, repl);
+
+    final int newLength = blockSize;
+
+    boolean isReady = fc.truncate(file, newLength);
+
+    Assert.assertTrue("Recovery is not expected.", isReady);
+
+    FileStatus fileStatus = fc.getFileStatus(file);
+    Assert.assertEquals(fileStatus.getLen(), newLength);
+    AppendTestUtil.checkFullFile(fs, file, newLength, data, file.toString());
+
+    ContentSummary cs = fs.getContentSummary(dir);
+    Assert.assertEquals("Bad disk space usage", cs.getSpaceConsumed(),
+        newLength * repl);
+    Assert.assertTrue(fs.delete(dir, true));
+  }
+
   @Test
   public void testOldRenameWithQuota() throws Exception {
     DistributedFileSystem fs = cluster.getFileSystem();


Mime
View raw message