Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F41571071D for ; Mon, 21 Oct 2013 15:48:38 +0000 (UTC) Received: (qmail 77089 invoked by uid 500); 21 Oct 2013 15:48:38 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 76924 invoked by uid 500); 21 Oct 2013 15:48:32 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 76906 invoked by uid 99); 21 Oct 2013 15:48:30 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Oct 2013 15:48:30 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Oct 2013 15:48:28 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id CCC1D2388831; Mon, 21 Oct 2013 15:48:08 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1534240 - in /hive/trunk/shims/src: 0.20/java/org/apache/hadoop/hive/shims/ 0.20S/java/org/apache/hadoop/hive/shims/ 0.23/java/org/apache/hadoop/hive/shims/ common-secure/java/org/apache/hadoop/hive/shims/ common/java/org/apache/hadoop/fs/... Date: Mon, 21 Oct 2013 15:48:08 -0000 To: commits@hive.apache.org From: brock@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131021154808.CCC1D2388831@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: brock Date: Mon Oct 21 15:48:08 2013 New Revision: 1534240 URL: http://svn.apache.org/r1534240 Log: HIVE-5070 - Implement listLocatedStatus() in ProxyFileSystem for 0.23 shim (shanyu zhao via Brock Noland) Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1534240&r1=1534239&r2=1534240&view=diff ============================================================================== --- hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original) +++ hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Mon Oct 21 15:48:08 2013 @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.ProxyFileSystem; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; @@ -783,4 +784,8 @@ public class Hadoop20Shims implements Ha public WebHCatJTShim getWebHCatShim(Configuration conf, UserGroupInformation ugi) throws IOException { throw new UnsupportedOperationException("WebHCat does not support Hadoop 0.20.x"); } + @Override + public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { + return new ProxyFileSystem(fs, uri); + } } Modified: hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1534240&r1=1534239&r2=1534240&view=diff ============================================================================== --- hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original) +++ hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Mon Oct 21 15:48:08 2013 @@ -22,6 +22,7 @@ import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URL; import java.util.Iterator; +import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; @@ -30,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.ProxyFileSystem; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapred.JobTracker; @@ -372,4 +374,8 @@ public class Hadoop20SShims extends Hado return fs.getFileBlockLocations(status, 0, status.getLen()); } + @Override + public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { + return new ProxyFileSystem(fs, uri); + } } Modified: hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1534240&r1=1534239&r2=1534240&view=diff ============================================================================== --- hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original) +++ hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Mon Oct 21 15:48:08 2013 @@ -24,6 +24,8 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.Iterator; import java.util.Map; +import java.net.URI; +import java.io.FileNotFoundException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -33,6 +35,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.fs.ProxyFileSystem; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -407,4 +412,40 @@ public class Hadoop23Shims extends Hadoo } } + class ProxyFileSystem23 extends ProxyFileSystem { + public ProxyFileSystem23(FileSystem fs) { + super(fs); + } + public ProxyFileSystem23(FileSystem fs, URI uri) { + super(fs, uri); + } + + @Override + public RemoteIterator listLocatedStatus(final Path f) + throws FileNotFoundException, IOException { + return new RemoteIterator() { + private RemoteIterator stats = + ProxyFileSystem23.super.listLocatedStatus( + ProxyFileSystem23.super.swizzleParamPath(f)); + + @Override + public boolean hasNext() throws IOException { + return stats.hasNext(); + } + + @Override + public LocatedFileStatus next() throws IOException { + LocatedFileStatus result = stats.next(); + return new LocatedFileStatus( + ProxyFileSystem23.super.swizzleFileStatus(result, false), + result.getBlockLocations()); + } + }; + } + } + + @Override + public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { + return new ProxyFileSystem23(fs, uri); + } } Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1534240&r1=1534239&r2=1534240&view=diff ============================================================================== --- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original) +++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Mon Oct 21 15:48:08 2013 @@ -622,4 +622,7 @@ public abstract class HadoopShimsSecure @Override abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf) throws IOException; + + @Override + abstract public FileSystem createProxyFileSystem(FileSystem fs, URI uri); } Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java?rev=1534240&r1=1534239&r2=1534240&view=diff ============================================================================== --- hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java (original) +++ hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java Mon Oct 21 15:48:08 2013 @@ -45,7 +45,7 @@ public class ProxyFileSystem extends Fil - private Path swizzleParamPath(Path p) { + protected Path swizzleParamPath(Path p) { String pathUriString = p.toUri().toString(); URI newPathUri = URI.create(pathUriString); return new Path (realScheme, realAuthority, newPathUri.getPath()); @@ -57,7 +57,7 @@ public class ProxyFileSystem extends Fil return new Path (myScheme, myAuthority, newPathUri.getPath()); } - private FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) { + protected FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) { FileStatus ret = new FileStatus(orig.getLen(), orig.isDir(), orig.getReplication(), orig.getBlockSize(), orig.getModificationTime(), Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java?rev=1534240&r1=1534239&r2=1534240&view=diff ============================================================================== --- hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java (original) +++ hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java Mon Oct 21 15:48:08 2013 @@ -23,6 +23,8 @@ import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.hive.shims.HadoopShims; /**************************************************************** * A Proxy for LocalFileSystem @@ -61,7 +63,9 @@ public class ProxyLocalFileSystem extend String authority = name.getAuthority() != null ? name.getAuthority() : ""; String proxyUriString = nameUriString + "://" + authority + "/"; - fs = new ProxyFileSystem(localFs, URI.create(proxyUriString)); + + fs = ShimLoader.getHadoopShims().createProxyFileSystem( + localFs, URI.create(proxyUriString)); fs.initialize(name, conf); } Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1534240&r1=1534239&r2=1534240&view=diff ============================================================================== --- hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original) +++ hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Mon Oct 21 15:48:08 2013 @@ -562,4 +562,10 @@ public interface HadoopShims { */ public void close(); } + + /** + * Create a proxy file system that can serve a given scheme/authority using some + * other file system. + */ + public FileSystem createProxyFileSystem(FileSystem fs, URI uri); }