hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1417267 - in /hive/branches/branch-0.9: metastore/src/java/org/apache/hadoop/hive/metastore/ shims/src/0.20/java/org/apache/hadoop/hive/shims/ shims/src/0.20S/java/org/apache/hadoop/hive/shims/ shims/src/0.23/java/org/apache/hadoop/hive/sh...
Date Wed, 05 Dec 2012 01:59:23 GMT
Author: hashutosh
Date: Wed Dec  5 01:59:19 2012
New Revision: 1417267

URL: http://svn.apache.org/viewvc?rev=1417267&view=rev
Log:
HIVE-3648 : HiveMetaStoreFsImpl is not compatible with hadoop viewfs (Arup Malakar via Ashutosh
Chauhan)

Modified:
    hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
    hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
    hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
    hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java

Modified: hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
(original)
+++ hive/branches/branch-0.9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
Wed Dec  5 01:59:19 2012
@@ -25,8 +25,9 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
 
 public class HiveMetaStoreFsImpl implements MetaStoreFS {
 
@@ -37,16 +38,10 @@ public class HiveMetaStoreFsImpl impleme
   public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
       Configuration conf) throws MetaException {
     LOG.info("deleting  " + f);
-
-    // older versions of Hadoop don't have a Trash constructor based on the
-    // Path or FileSystem. So need to achieve this by creating a dummy conf.
-    // this needs to be filtered out based on version
-    Configuration dupConf = new Configuration(conf);
-    FileSystem.setDefaultUri(dupConf, fs.getUri());
+    HadoopShims hadoopShim = ShimLoader.getHadoopShims();
 
     try {
-      Trash trashTmp = new Trash(dupConf);
-      if (trashTmp.moveToTrash(f)) {
+      if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) {
         LOG.info("Moved to trash: " + f);
         return true;
       }

Modified: hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
(original)
+++ hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
Wed Dec  5 01:59:19 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
 import org.apache.hadoop.io.Text;
@@ -603,4 +604,17 @@ public class Hadoop20Shims implements Ha
     // No such functionality in ancient hadoop
     return;
   }
+
+  @Override
+  public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+          throws IOException {
+    // older versions of Hadoop don't have a Trash constructor based on the
+    // Path or FileSystem. So need to achieve this by creating a dummy conf.
+    // this needs to be filtered out based on version
+
+    Configuration dupConf = new Configuration(conf);
+    FileSystem.setDefaultUri(dupConf, fs.getUri());
+    Trash trash = new Trash(dupConf);
+    return trash.moveToTrash(path);
+  }
 }

Modified: hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
(original)
+++ hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Wed Dec  5 01:59:19 2012
@@ -17,12 +17,14 @@
  */
 package org.apache.hadoop.hive.shims;
 
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.hive.shims.HadoopShimsSecure;
 import org.apache.hadoop.mapred.ClusterStatus;
 import org.apache.hadoop.mapreduce.Job;
@@ -97,6 +99,19 @@ public class Hadoop20SShims extends Hado
   }
 
   @Override
+  public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+          throws IOException {
+    // older versions of Hadoop don't have a Trash constructor based on the
+    // Path or FileSystem. So need to achieve this by creating a dummy conf.
+    // this needs to be filtered out based on version
+
+    Configuration dupConf = new Configuration(conf);
+    FileSystem.setDefaultUri(dupConf, fs.getUri());
+    Trash trash = new Trash(dupConf);
+    return trash.moveToTrash(path);
+  }
+  
+  @Override
   public long getDefaultBlockSize(FileSystem fs, Path path) {
     return fs.getDefaultBlockSize();
   }
@@ -104,5 +119,5 @@ public class Hadoop20SShims extends Hado
   @Override
   public short getDefaultReplication(FileSystem fs, Path path) {
     return fs.getDefaultReplication();
-  }
+  }  
 }

Modified: hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
(original)
+++ hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
Wed Dec  5 01:59:19 2012
@@ -17,12 +17,14 @@
  */
 package org.apache.hadoop.hive.shims;
 
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
 import org.apache.hadoop.hive.shims.HadoopShims.JobTrackerState;
 import org.apache.hadoop.mapred.ClusterStatus;
 import org.apache.hadoop.mapred.JobConf;
@@ -123,4 +125,9 @@ public class Hadoop23Shims extends Hadoo
     return fs.getDefaultReplication(path);
   }
 
+  @Override
+  public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+          throws IOException {
+    return Trash.moveToAppropriateTrash(fs, path, conf);
+  }
 }

Modified: hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
(original)
+++ hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
Wed Dec  5 01:59:19 2012
@@ -555,6 +555,10 @@ public abstract class HadoopShimsSecure 
   abstract public String getJobLauncherRpcAddress(Configuration conf);
 
   @Override
+  abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration
conf)
+          throws IOException;
+
+  @Override
   abstract public short getDefaultReplication(FileSystem fs, Path path);
 
   @Override

Modified: hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1417267&r1=1417266&r2=1417267&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
(original)
+++ hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Wed Dec  5 01:59:19 2012
@@ -275,7 +275,7 @@ public interface HadoopShims {
    * @return
    */
   public String getJobLauncherHttpAddress(Configuration conf);
-
+  
   /**
    * Get the default block size for the path. FileSystem alone is not sufficient to
    * determine the same, as in case of CSMT the underlying file system determines that.
@@ -293,11 +293,22 @@ public interface HadoopShims {
    * @return
    */
   public short getDefaultReplication(FileSystem fs, Path path);
-
   /**
-   * InputSplitShim.
-   *
+   * Move the directory/file to trash. In case of the symlinks or mount points, the file
is
+   * moved to the trashbin in the actual volume of the path p being deleted
+   * @param fs
+   * @param path
+   * @param conf
+   * @return false if the item is already in the trash or trash is disabled
+   * @throws IOException
    */
+  public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf)
+          throws IOException;
+
+ /**
+  * InputSplitShim.
+  *
+  */
   public interface InputSplitShim extends InputSplit {
     JobConf getJob();
 



Mime
View raw message