Return-Path: Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: (qmail 29889 invoked from network); 2 Nov 2009 21:46:07 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 2 Nov 2009 21:46:07 -0000 Received: (qmail 76070 invoked by uid 500); 2 Nov 2009 21:46:07 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 76020 invoked by uid 500); 2 Nov 2009 21:46:07 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 75999 invoked by uid 99); 2 Nov 2009 21:46:07 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Nov 2009 21:46:07 +0000 X-ASF-Spam-Status: No, hits=-2.6 required=5.0 tests=BAYES_00 X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Nov 2009 21:46:04 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 32802238888F; Mon, 2 Nov 2009 21:45:44 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r832118 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/hdfs/ src/test/hdfs/org/apache/hadoop/fs/ src/test/hdfs/org/apache/hadoop/hdfs/ Date: Mon, 02 Nov 2009 21:45:43 -0000 To: hdfs-commits@hadoop.apache.org From: suresh@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20091102214544.32802238888F@eris.apache.org> Author: suresh Date: Mon Nov 2 21:45:43 2009 New Revision: 832118 URL: http://svn.apache.org/viewvc?rev=832118&view=rev Log: HDFS-702. Add HDFS implementation of AbstractFileSystem. Contributed by Sanjay Radia. Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/fs/ hadoop/hdfs/trunk/src/java/org/apache/hadoop/fs/Hdfs.java hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsCreateMkdir.java hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsPermission.java Modified: hadoop/hdfs/trunk/CHANGES.txt hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java Modified: hadoop/hdfs/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=832118&r1=832117&r2=832118&view=diff ============================================================================== --- hadoop/hdfs/trunk/CHANGES.txt (original) +++ hadoop/hdfs/trunk/CHANGES.txt Mon Nov 2 21:45:43 2009 @@ -139,6 +139,9 @@ HDFS-731. Support new Syncable interface in HDFS. (hairong) + HDFS-702. Add HDFS implementation of AbstractFileSystem. + (Sanjay Radio via suresh) + IMPROVEMENTS HDFS-381. Remove blocks from DataNode maps when corresponding file Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/fs/Hdfs.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/fs/Hdfs.java?rev=832118&view=auto ============================================================================== --- hadoop/hdfs/trunk/src/java/org/apache/hadoop/fs/Hdfs.java (added) +++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/fs/Hdfs.java Mon Nov 2 21:45:43 2009 @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.EnumSet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.protocol.FSConstants; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.util.Progressable; + +public class Hdfs extends AbstractFileSystem { + + DFSClient dfs; + private boolean verifyChecksum = true; + + static { + Configuration.addDefaultResource("hdfs-default.xml"); + Configuration.addDefaultResource("hdfs-site.xml"); + } + + /** + * This constructor has the signature needed by + * {@link AbstractFileSystem#createFileSystem(URI, Configuration)} + * + * @param theUri + * which must be that of Hdfs + * @param conf + * @throws IOException + */ + Hdfs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException { + super(theUri, FSConstants.HDFS_URI_SCHEME, true, NameNode.DEFAULT_PORT); + + if (!theUri.getScheme().equalsIgnoreCase(FSConstants.HDFS_URI_SCHEME)) { + throw new IllegalArgumentException("Passed URI's scheme is not for Hdfs"); + } + String host = theUri.getHost(); + if (host == null) { + throw new IOException("Incomplete HDFS URI, no host: " + theUri); + } + + InetSocketAddress namenode = NameNode.getAddress(theUri.getAuthority()); + this.dfs = new DFSClient(namenode, conf, getStatistics()); + } + + @Override + protected int getUriDefaultPort() { + return NameNode.DEFAULT_PORT; + } + + @Override + protected FSDataOutputStream createInternal(Path f, + EnumSet createFlag, FsPermission absolutePermission, + int bufferSize, short replication, long blockSize, Progressable progress, + int bytesPerChecksum, boolean createParent) throws IOException { + return new FSDataOutputStream(dfs.primitiveCreate(getUriPath(f), + absolutePermission, createFlag, createParent, replication, blockSize, + progress, bufferSize, bytesPerChecksum), getStatistics()); + } + + @Override + protected boolean delete(Path f, boolean recursive) throws IOException { + return dfs.delete(getUriPath(f), recursive); + } + + @Override + protected BlockLocation[] getFileBlockLocations(Path p, long start, long len) + throws IOException { + return dfs.getBlockLocations(p.toString(), start, len); + } + + @Override + protected FileChecksum getFileChecksum(Path f) throws IOException { + return dfs.getFileChecksum(getUriPath(f)); + } + + @Override + protected FileStatus getFileStatus(Path f) throws IOException { + FileStatus fi = dfs.getFileInfo(getUriPath(f)); + if (fi != null) { + fi.setPath(fi.getPath().makeQualified(getUri(), null)); + return fi; + } else { + throw new FileNotFoundException("File does not exist: " + f.toString()); + } + } + + @Override + protected FsStatus getFsStatus() throws IOException { + return dfs.getDiskStatus(); + } + + @Override + protected FsServerDefaults getServerDefaults() throws IOException { + return dfs.getServerDefaults(); + } + + @Override + protected FileStatus[] listStatus(Path f) throws IOException { + FileStatus[] infos = dfs.listPaths(getUriPath(f)); + if (infos == null) + throw new FileNotFoundException("File " + f + " does not exist."); + + for (int i = 0; i < infos.length; i++) { + infos[i].setPath(infos[i].getPath().makeQualified(getUri(), null)); + } + return infos; + } + + @Override + protected void mkdir(Path dir, FsPermission permission, boolean createParent) + throws IOException { + dfs.mkdirs(getUriPath(dir), permission, createParent); + + } + + @Override + protected FSDataInputStream open(Path f, int bufferSize) throws IOException { + return new DFSClient.DFSDataInputStream(dfs.open(getUriPath(f), + bufferSize, verifyChecksum)); + } + + @Override + protected void renameInternal(Path src, Path dst) throws IOException { + dfs.rename(getUriPath(src), getUriPath(dst)); + } + + @Override + protected void renameInternal(Path src, Path dst, boolean overwrite) + throws IOException { + dfs.rename(getUriPath(src), getUriPath(dst), + overwrite ? Options.Rename.OVERWRITE : Options.Rename.NONE); + } + + @Override + protected void setOwner(Path f, String username, String groupname) + throws IOException { + dfs.setOwner(getUriPath(f), username, groupname); + + } + + @Override + protected void setPermission(Path f, FsPermission permission) + throws IOException { + dfs.setPermission(getUriPath(f), permission); + + } + + @Override + protected boolean setReplication(Path f, short replication) + throws IOException { + return dfs.setReplication(getUriPath(f), replication); + } + + @Override + protected void setTimes(Path f, long mtime, long atime) throws IOException { + dfs.setTimes(getUriPath(f), mtime, atime); + + } + + @Override + protected void setVerifyChecksum(boolean verifyChecksum) throws IOException { + this.verifyChecksum = verifyChecksum; + } +} Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java?rev=832118&r1=832117&r2=832118&view=diff ============================================================================== --- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClient.java Mon Nov 2 21:45:43 2009 @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -418,10 +419,24 @@ * namenode, and then reads from all the right places. Creates * inner subclass of InputStream that does the right out-of-band * work. + * @deprecated Use {@link #open(String, int, boolean)} instead. */ - DFSInputStream open(String src, int buffersize, boolean verifyChecksum, + @Deprecated + public DFSInputStream open(String src, int buffersize, boolean verifyChecksum, FileSystem.Statistics stats ) throws IOException { + return open(src, buffersize, verifyChecksum); + } + + + /** + * Create an input stream that obtains a nodelist from the + * namenode, and then reads from all the right places. Creates + * inner subclass of InputStream that does the right out-of-band + * work. + */ + public DFSInputStream open(String src, int buffersize, boolean verifyChecksum + ) throws IOException { checkOpen(); // Get block info from namenode return new DFSInputStream(src, buffersize, verifyChecksum); @@ -594,11 +609,6 @@ int bytesPerChecksum) throws IOException { checkOpen(); - if (absPermission == null) { - absPermission = - FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf)); - } - LOG.debug(src + ": masked=" + absPermission); OutputStream result = new DFSOutputStream(src, absPermission, flag, createParent, replication, blockSize, progress, buffersize, bytesPerChecksum); @@ -757,7 +767,7 @@ * @return The checksum * @see DistributedFileSystem#getFileChecksum(Path) */ - MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException { + public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException { checkOpen(); return getFileChecksum(src, namenode, socketFactory, socketTimeout); } @@ -2374,8 +2384,8 @@ } } - static class DFSDataInputStream extends FSDataInputStream { - DFSDataInputStream(DFSInputStream in) + public static class DFSDataInputStream extends FSDataInputStream { + public DFSDataInputStream(DFSInputStream in) throws IOException { super(in); } Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=832118&r1=832117&r2=832118&view=diff ============================================================================== --- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original) +++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Mon Nov 2 21:45:43 2009 @@ -191,6 +191,7 @@ this.verifyChecksum = verifyChecksum; } + @SuppressWarnings("deprecation") @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { return new DFSClient.DFSDataInputStream( Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsCreateMkdir.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsCreateMkdir.java?rev=832118&view=auto ============================================================================== --- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsCreateMkdir.java (added) +++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsCreateMkdir.java Mon Nov 2 21:45:43 2009 @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import java.io.IOException; +import java.net.URISyntaxException; + +import javax.security.auth.login.LoginException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.UnixUserGroupInformation; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +public class TestFcHdfsCreateMkdir extends + FileContextCreateMkdirBaseTest { + + private static MiniDFSCluster cluster; + private static Path defaultWorkingDirectory; + + @BeforeClass + public static void clusterSetupAtBegining() + throws IOException, LoginException, URISyntaxException { + Configuration conf = new HdfsConfiguration(); + cluster = new MiniDFSCluster(conf, 2, true, null); + fc = FileContext.getFileContext(cluster.getURI(), conf); + defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + + UnixUserGroupInformation.login().getUserName())); + fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true); + } + + + @AfterClass + public static void ClusterShutdownAtEnd() throws Exception { + cluster.shutdown(); + } + + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + } +} Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsPermission.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsPermission.java?rev=832118&view=auto ============================================================================== --- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsPermission.java (added) +++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsPermission.java Mon Nov 2 21:45:43 2009 @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import java.io.IOException; +import java.net.URISyntaxException; + +import javax.security.auth.login.LoginException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.UnixUserGroupInformation; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +public class TestFcHdfsPermission extends FileContextPermissionBase { + + private static MiniDFSCluster cluster; + private static Path defaultWorkingDirectory; + + @BeforeClass + public static void clusterSetupAtBegining() + throws IOException, LoginException, URISyntaxException { + Configuration conf = new HdfsConfiguration(); + cluster = new MiniDFSCluster(conf, 2, true, null); + fc = FileContext.getFileContext(cluster.getURI(), conf); + defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + + UnixUserGroupInformation.login().getUserName())); + fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true); + } + + + @AfterClass + public static void ClusterShutdownAtEnd() throws Exception { + cluster.shutdown(); + } + + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + } + + /* + * HDFS ignore the "x" bit if the permission. + * + */ + static final FsPermission FILE_MASK_IGNORE_X_BIT = + new FsPermission((short) ~0666); + FsPermission getFileMask() { + return FILE_MASK_IGNORE_X_BIT; + } +} Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java?rev=832118&r1=832117&r2=832118&view=diff ============================================================================== --- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java (original) +++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/fs/TestHDFSFileContextMainOperations.java Mon Nov 2 21:45:43 2009 @@ -19,9 +19,11 @@ package org.apache.hadoop.fs; import java.io.IOException; +import java.net.URISyntaxException; import javax.security.auth.login.LoginException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -42,10 +44,11 @@ private static Path defaultWorkingDirectory; @BeforeClass - public static void clusterSetupAtBegining() - throws IOException, LoginException { - cluster = new MiniDFSCluster(new HdfsConfiguration(), 2, true, null); - fc = FileContext.getFileContext(cluster.getFileSystem()); + public static void clusterSetupAtBegining() throws IOException, + LoginException, URISyntaxException { + Configuration conf = new HdfsConfiguration(); + cluster = new MiniDFSCluster(conf, 2, true, null); + fc = FileContext.getFileContext(cluster.getURI(), conf); defaultWorkingDirectory = fc.makeQualified( new Path("/user/" + UnixUserGroupInformation.login().getUserName())); fc.mkdir(defaultWorkingDirectory, FileContext.DEFAULT_PERM, true); Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=832118&r1=832117&r2=832118&view=diff ============================================================================== --- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java (original) +++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java Mon Nov 2 21:45:43 2009 @@ -22,6 +22,7 @@ import java.io.RandomAccessFile; import java.net.InetSocketAddress; import java.net.URI; +import java.net.URISyntaxException; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Collection; @@ -71,7 +72,7 @@ this.dnArgs = args; } } - + private URI myUri = null; private Configuration conf; private NameNode nameNode; private int numDataNodes; @@ -289,6 +290,20 @@ startDataNodes(conf, numDataNodes, manageDataDfsDirs, operation, racks, hosts, simulatedCapacities); waitClusterUp(); + String myUriStr = "hdfs://localhost:"+ Integer.toString(this.getNameNodePort()); + try { + this.myUri = new URI(myUriStr); + } catch (URISyntaxException e) { + NameNode.LOG.warn("unexpected URISyntaxException: " + e ); + } + } + + /** + * + * @return URI of this MiniDFSCluster + */ + public URI getURI() { + return myUri; } /**