hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject svn commit: r1496691 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs: TestFcHdfsSymlink.java TestSymlinkHdfs.java TestSymlinkHdfsFileContext.java
Date Wed, 26 Jun 2013 00:36:02 GMT
Author: cmccabe
Date: Wed Jun 26 00:36:01 2013
New Revision: 1496691

URL: http://svn.apache.org/r1496691
Log:
HADOOP-9355. Abstract Symlink tests to use either FileContext or FileSystem (Andrew Wang via
Colin Patrick McCabe)

Added:
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java
Removed:
    hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java

Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java?rev=1496691&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java
(added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfs.java
Wed Jun 26 00:36:01 2013
@@ -0,0 +1,340 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test symbolic links in Hdfs.
+ */
+abstract public class TestSymlinkHdfs extends SymlinkBaseTest {
+
+  {
+    ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL);
+  }
+
+  protected static MiniDFSCluster cluster;
+  protected static WebHdfsFileSystem webhdfs;
+  protected static DistributedFileSystem dfs;
+
+  @Override
+  protected String getScheme() {
+    return "hdfs";
+  }
+
+  @Override
+  protected String testBaseDir1() throws IOException {
+    return "/test1";
+  }
+  
+  @Override
+  protected String testBaseDir2() throws IOException {
+    return "/test2";
+  }
+
+  @Override
+  protected URI testURI() {
+    return cluster.getURI(0);
+  }
+
+  @Override
+  protected IOException unwrapException(IOException e) {
+    if (e instanceof RemoteException) {
+      return ((RemoteException)e).unwrapRemoteException();
+    }
+    return e;
+  }
+
+  @BeforeClass
+  public static void beforeClassSetup() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
+    conf.set(FsPermission.UMASK_LABEL, "000");
+    cluster = new MiniDFSCluster.Builder(conf).build();
+    webhdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf);
+    dfs = cluster.getFileSystem();
+  }
+
+  @AfterClass
+  public static void afterClassTeardown() throws Exception {
+    cluster.shutdown();
+  }
+
+  @Test(timeout=10000)
+  /** Access a file using a link that spans Hdfs to LocalFs */
+  public void testLinkAcrossFileSystems() throws IOException {
+    Path localDir = new Path("file://" + wrapper.getAbsoluteTestRootDir()
+        + "/test");
+    Path localFile = new Path("file://" + wrapper.getAbsoluteTestRootDir()
+        + "/test/file");
+    Path link      = new Path(testBaseDir1(), "linkToFile");
+    FSTestWrapper localWrapper = wrapper.getLocalFSWrapper();
+    localWrapper.delete(localDir, true);
+    localWrapper.mkdir(localDir, FileContext.DEFAULT_PERM, true);
+    localWrapper.setWorkingDirectory(localDir);
+    assertEquals(localDir, localWrapper.getWorkingDirectory());
+    createAndWriteFile(localWrapper, localFile);
+    wrapper.createSymlink(localFile, link, false);
+    readFile(link);
+    assertEquals(fileSize, wrapper.getFileStatus(link).getLen());
+  }
+
+  @Test(timeout=10000)
+  /** Test renaming a file across two file systems using a link */
+  public void testRenameAcrossFileSystemsViaLink() throws IOException {
+    Path localDir = new Path("file://" + wrapper.getAbsoluteTestRootDir()
+        + "/test");
+    Path hdfsFile    = new Path(testBaseDir1(), "file");
+    Path link        = new Path(testBaseDir1(), "link");
+    Path hdfsFileNew = new Path(testBaseDir1(), "fileNew");
+    Path hdfsFileNewViaLink = new Path(link, "fileNew");
+    FSTestWrapper localWrapper = wrapper.getLocalFSWrapper();
+    localWrapper.delete(localDir, true);
+    localWrapper.mkdir(localDir, FileContext.DEFAULT_PERM, true);
+    localWrapper.setWorkingDirectory(localDir);
+    createAndWriteFile(hdfsFile);
+    wrapper.createSymlink(localDir, link, false);
+    // Rename hdfs://test1/file to hdfs://test1/link/fileNew
+    // which renames to file://TEST_ROOT/test/fileNew which
+    // spans AbstractFileSystems and therefore fails.
+    try {
+      wrapper.rename(hdfsFile, hdfsFileNewViaLink);
+      fail("Renamed across file systems");
+    } catch (InvalidPathException ipe) {
+      // Expected from FileContext
+    } catch (IllegalArgumentException e) {
+      // Expected from Filesystem
+      GenericTestUtils.assertExceptionContains("Wrong FS: ", e);
+    }
+    // Now rename hdfs://test1/link/fileNew to hdfs://test1/fileNew
+    // which renames file://TEST_ROOT/test/fileNew to hdfs://test1/fileNew
+    // which spans AbstractFileSystems and therefore fails.
+    createAndWriteFile(hdfsFileNewViaLink);
+    try {
+      wrapper.rename(hdfsFileNewViaLink, hdfsFileNew);
+      fail("Renamed across file systems");
+    } catch (InvalidPathException ipe) {
+      // Expected from FileContext
+    } catch (IllegalArgumentException e) {
+      // Expected from Filesystem
+      GenericTestUtils.assertExceptionContains("Wrong FS: ", e);
+    }
+  }
+
+  @Test(timeout=10000)
+  /** Test create symlink to / */
+  public void testCreateLinkToSlash() throws IOException {
+    Path dir  = new Path(testBaseDir1());
+    Path file = new Path(testBaseDir1(), "file");
+    Path link = new Path(testBaseDir1(), "linkToSlash");
+    Path fileViaLink = new Path(testBaseDir1()+"/linkToSlash"+
+                                testBaseDir1()+"/file");
+    createAndWriteFile(file);
+    wrapper.setWorkingDirectory(dir);
+    wrapper.createSymlink(new Path("/"), link, false);
+    readFile(fileViaLink);
+    assertEquals(fileSize, wrapper.getFileStatus(fileViaLink).getLen());
+    // Ditto when using another file context since the file system
+    // for the slash is resolved according to the link's parent.
+    if (wrapper instanceof FileContextTestWrapper) {
+      FSTestWrapper localWrapper = wrapper.getLocalFSWrapper();
+      Path linkQual = new Path(cluster.getURI(0).toString(), fileViaLink);
+      assertEquals(fileSize, localWrapper.getFileStatus(linkQual).getLen());
+    }
+  }
+  
+  
+  @Test(timeout=10000)
+  /** setPermission affects the target not the link */
+  public void testSetPermissionAffectsTarget() throws IOException {
+    Path file       = new Path(testBaseDir1(), "file");
+    Path dir        = new Path(testBaseDir2());
+    Path linkToFile = new Path(testBaseDir1(), "linkToFile");
+    Path linkToDir  = new Path(testBaseDir1(), "linkToDir");
+    createAndWriteFile(file);
+    wrapper.createSymlink(file, linkToFile, false);
+    wrapper.createSymlink(dir, linkToDir, false);
+    
+    // Changing the permissions using the link does not modify
+    // the permissions of the link..
+    FsPermission perms = wrapper.getFileLinkStatus(linkToFile).getPermission();
+    wrapper.setPermission(linkToFile, new FsPermission((short)0664));
+    wrapper.setOwner(linkToFile, "user", "group");
+    assertEquals(perms, wrapper.getFileLinkStatus(linkToFile).getPermission());
+    // but the file's permissions were adjusted appropriately
+    FileStatus stat = wrapper.getFileStatus(file);
+    assertEquals(0664, stat.getPermission().toShort());
+    assertEquals("user", stat.getOwner());
+    assertEquals("group", stat.getGroup());
+    // Getting the file's permissions via the link is the same
+    // as getting the permissions directly.
+    assertEquals(stat.getPermission(), 
+                 wrapper.getFileStatus(linkToFile).getPermission());
+
+    // Ditto for a link to a directory
+    perms = wrapper.getFileLinkStatus(linkToDir).getPermission();
+    wrapper.setPermission(linkToDir, new FsPermission((short)0664));
+    wrapper.setOwner(linkToDir, "user", "group");
+    assertEquals(perms, wrapper.getFileLinkStatus(linkToDir).getPermission());
+    stat = wrapper.getFileStatus(dir);
+    assertEquals(0664, stat.getPermission().toShort());
+    assertEquals("user", stat.getOwner());
+    assertEquals("group", stat.getGroup());
+    assertEquals(stat.getPermission(), 
+                 wrapper.getFileStatus(linkToDir).getPermission());
+  }  
+
+  @Test(timeout=10000)
+  /** Create a symlink using a path with scheme but no authority */
+  public void testCreateWithPartQualPathFails() throws IOException {
+    Path fileWoAuth = new Path("hdfs:///test/file");
+    Path linkWoAuth = new Path("hdfs:///test/link");
+    try {
+      createAndWriteFile(fileWoAuth);
+      fail("HDFS requires URIs with schemes have an authority");
+    } catch (RuntimeException e) {
+      // Expected
+    }
+    try {
+      wrapper.createSymlink(new Path("foo"), linkWoAuth, false);
+      fail("HDFS requires URIs with schemes have an authority");
+    } catch (RuntimeException e) {
+      // Expected
+    }
+  }
+
+  @Test(timeout=10000)
+  /** setReplication affects the target not the link */
+  public void testSetReplication() throws IOException {
+    Path file = new Path(testBaseDir1(), "file");
+    Path link = new Path(testBaseDir1(), "linkToFile");
+    createAndWriteFile(file);
+    wrapper.createSymlink(file, link, false);
+    wrapper.setReplication(link, (short)2);
+    assertEquals(0, wrapper.getFileLinkStatus(link).getReplication());
+    assertEquals(2, wrapper.getFileStatus(link).getReplication());
+    assertEquals(2, wrapper.getFileStatus(file).getReplication());
+  }
+  
+  @Test(timeout=10000)
+  /** Test create symlink with a max len name */
+  public void testCreateLinkMaxPathLink() throws IOException {
+    Path dir  = new Path(testBaseDir1());
+    Path file = new Path(testBaseDir1(), "file");
+    final int maxPathLen = HdfsConstants.MAX_PATH_LENGTH;
+    final int dirLen     = dir.toString().length() + 1;
+    int   len            = maxPathLen - dirLen;
+    
+    // Build a MAX_PATH_LENGTH path
+    StringBuilder sb = new StringBuilder("");
+    for (int i = 0; i < (len / 10); i++) {
+      sb.append("0123456789");
+    }
+    for (int i = 0; i < (len % 10); i++) {
+      sb.append("x");
+    }
+    Path link = new Path(sb.toString());
+    assertEquals(maxPathLen, dirLen + link.toString().length()); 
+    
+    // Check that it works
+    createAndWriteFile(file);
+    wrapper.setWorkingDirectory(dir);
+    wrapper.createSymlink(file, link, false);
+    readFile(link);
+    
+    // Now modify the path so it's too large
+    link = new Path(sb.toString()+"x");
+    try {
+      wrapper.createSymlink(file, link, false);
+      fail("Path name should be too long");
+    } catch (IOException x) {
+      // Expected
+    }
+  }
+
+  @Test(timeout=10000)
+  /** Test symlink owner */
+  public void testLinkOwner() throws IOException {
+    Path file = new Path(testBaseDir1(), "file");
+    Path link = new Path(testBaseDir1(), "symlinkToFile");
+    createAndWriteFile(file);
+    wrapper.createSymlink(file, link, false);
+    FileStatus statFile = wrapper.getFileStatus(file);
+    FileStatus statLink = wrapper.getFileStatus(link);
+    assertEquals(statLink.getOwner(), statFile.getOwner());
+  }
+
+  @Test(timeout=10000)
+  /** Test WebHdfsFileSystem.createSymlink(..). */
+  public void testWebHDFS() throws IOException {
+    Path file = new Path(testBaseDir1(), "file");
+    Path link = new Path(testBaseDir1(), "linkToFile");
+    createAndWriteFile(file);
+    webhdfs.createSymlink(file, link, false);
+    wrapper.setReplication(link, (short)2);
+    assertEquals(0, wrapper.getFileLinkStatus(link).getReplication());
+    assertEquals(2, wrapper.getFileStatus(link).getReplication());
+    assertEquals(2, wrapper.getFileStatus(file).getReplication());
+  }
+
+  @Test(timeout=10000)
+  /** Test craeteSymlink(..) with quota. */
+  public void testQuota() throws IOException {
+    final Path dir = new Path(testBaseDir1());
+    dfs.setQuota(dir, 3, HdfsConstants.QUOTA_DONT_SET);
+
+    final Path file = new Path(dir, "file");
+    createAndWriteFile(file);
+
+    //creating the first link should succeed
+    final Path link1 = new Path(dir, "link1");
+    wrapper.createSymlink(file, link1, false);
+
+    try {
+      //creating the second link should fail with QuotaExceededException.
+      final Path link2 = new Path(dir, "link2");
+      wrapper.createSymlink(file, link2, false);
+      fail("Created symlink despite quota violation");
+    } catch(QuotaExceededException qee) {
+      //expected
+    }
+  }
+}

Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java?rev=1496691&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java
(added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileContext.java
Wed Jun 26 00:36:01 2013
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestSymlinkHdfsFileContext extends TestSymlinkHdfs {
+
+  private static FileContext fc;
+
+  @BeforeClass
+  public static void testSetup() throws Exception {
+    fc = FileContext.getFileContext(cluster.getURI(0));
+    wrapper = new FileContextTestWrapper(fc, "/tmp/TestSymlinkHdfsFileContext");
+  }
+
+  @Test(timeout=1000)
+  /** Test access a symlink using AbstractFileSystem */
+  public void testAccessLinkFromAbstractFileSystem() throws IOException {
+    Path file = new Path(testBaseDir1(), "file");
+    Path link = new Path(testBaseDir1(), "linkToFile");
+    createAndWriteFile(file);
+    wrapper.createSymlink(file, link, false);
+    try {
+      AbstractFileSystem afs = fc.getDefaultFileSystem();
+      afs.open(link);
+      fail("Opened a link using AFS");
+    } catch (UnresolvedLinkException x) {
+      // Expected
+    }
+  }
+}



Mime
View raw message