hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1534240 - in /hive/trunk/shims/src: 0.20/java/org/apache/hadoop/hive/shims/ 0.20S/java/org/apache/hadoop/hive/shims/ 0.23/java/org/apache/hadoop/hive/shims/ common-secure/java/org/apache/hadoop/hive/shims/ common/java/org/apache/hadoop/fs/...
Date Mon, 21 Oct 2013 15:48:08 GMT
Author: brock
Date: Mon Oct 21 15:48:08 2013
New Revision: 1534240

URL: http://svn.apache.org/r1534240
Log:
HIVE-5070 - Implement listLocatedStatus() in ProxyFileSystem for 0.23 shim (shanyu zhao via
Brock Noland)

Modified:
    hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
    hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java
    hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java
    hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java

Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Mon Oct
21 15:48:08 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
 import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
@@ -783,4 +784,8 @@ public class Hadoop20Shims implements Ha
   public WebHCatJTShim getWebHCatShim(Configuration conf, UserGroupInformation ugi) throws
IOException {
       throw new UnsupportedOperationException("WebHCat does not support Hadoop 0.20.x");
   }
+  @Override
+  public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+    return new ProxyFileSystem(fs, uri);
+  }
 }

Modified: hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Mon Oct
21 15:48:08 2013
@@ -22,6 +22,7 @@ import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Iterator;
+import java.net.URI;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
@@ -30,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
 import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapred.JobTracker;
@@ -372,4 +374,8 @@ public class Hadoop20SShims extends Hado
     return fs.getFileBlockLocations(status, 0, status.getLen());
   }
 
+  @Override
+  public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+    return new ProxyFileSystem(fs, uri);
+  }
 }

Modified: hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Mon Oct
21 15:48:08 2013
@@ -24,6 +24,8 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Iterator;
 import java.util.Map;
+import java.net.URI;
+import java.io.FileNotFoundException;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -33,6 +35,9 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.ProxyFileSystem;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -407,4 +412,40 @@ public class Hadoop23Shims extends Hadoo
     }
   }
 
+  class ProxyFileSystem23 extends ProxyFileSystem {
+    public ProxyFileSystem23(FileSystem fs) {
+      super(fs);
+    }
+    public ProxyFileSystem23(FileSystem fs, URI uri) {
+      super(fs, uri);
+    }
+
+    @Override
+    public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
+      throws FileNotFoundException, IOException {
+      return new RemoteIterator<LocatedFileStatus>() {
+        private RemoteIterator<LocatedFileStatus> stats =
+            ProxyFileSystem23.super.listLocatedStatus(
+                ProxyFileSystem23.super.swizzleParamPath(f));
+
+        @Override
+        public boolean hasNext() throws IOException {
+          return stats.hasNext();
+        }
+
+        @Override
+        public LocatedFileStatus next() throws IOException {
+          LocatedFileStatus result = stats.next();
+          return new LocatedFileStatus(
+              ProxyFileSystem23.super.swizzleFileStatus(result, false),
+              result.getBlockLocations());
+        }
+      };
+    }
+  }
+
+  @Override
+  public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
+    return new ProxyFileSystem23(fs, uri);
+  }
 }

Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
(original)
+++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
Mon Oct 21 15:48:08 2013
@@ -622,4 +622,7 @@ public abstract class HadoopShimsSecure 
   @Override
   abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration
conf)
           throws IOException;
+
+  @Override
+  abstract public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
 }

Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyFileSystem.java Mon Oct 21
15:48:08 2013
@@ -45,7 +45,7 @@ public class ProxyFileSystem extends Fil
 
 
 
-  private Path swizzleParamPath(Path p) {
+  protected Path swizzleParamPath(Path p) {
     String pathUriString = p.toUri().toString();
     URI newPathUri = URI.create(pathUriString);
     return new Path (realScheme, realAuthority, newPathUri.getPath());
@@ -57,7 +57,7 @@ public class ProxyFileSystem extends Fil
     return new Path (myScheme, myAuthority, newPathUri.getPath());
   }
 
-  private FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) {
+  protected FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) {
     FileStatus ret =
       new FileStatus(orig.getLen(), orig.isDir(), orig.getReplication(),
                      orig.getBlockSize(), orig.getModificationTime(),

Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java Mon Oct
21 15:48:08 2013
@@ -23,6 +23,8 @@ import java.net.URI;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.shims.HadoopShims;
 
 /****************************************************************
  * A Proxy for LocalFileSystem
@@ -61,7 +63,9 @@ public class ProxyLocalFileSystem extend
 
     String authority = name.getAuthority() != null ? name.getAuthority() : "";
     String proxyUriString = nameUriString + "://" + authority + "/";
-    fs = new ProxyFileSystem(localFs, URI.create(proxyUriString));
+
+    fs = ShimLoader.getHadoopShims().createProxyFileSystem(
+        localFs, URI.create(proxyUriString));
 
     fs.initialize(name, conf);
   }

Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1534240&r1=1534239&r2=1534240&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Mon Oct
21 15:48:08 2013
@@ -562,4 +562,10 @@ public interface HadoopShims {
      */
     public void close();
   }
+
+  /**
+   * Create a proxy file system that can serve a given scheme/authority using some
+   * other file system.
+   */
+  public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
 }



Mime
View raw message