hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1417220 - in /hive/branches/branch-0.9: ql/src/java/org/apache/hadoop/hive/ql/io/ shims/src/0.20/java/org/apache/hadoop/hive/shims/ shims/src/0.20S/java/org/apache/hadoop/hive/shims/ shims/src/0.23/java/org/apache/hadoop/hive/shims/ shims/...
Date Tue, 04 Dec 2012 22:12:28 GMT
Author: hashutosh
Date: Tue Dec  4 22:12:27 2012
New Revision: 1417220

URL: http://svn.apache.org/viewvc?rev=1417220&view=rev
Log:
HIVE-3645 : RCFileWriter does not implement the right function to support Federation (Arup
Malakar via Ashutosh Chauhan)

Modified:
    hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
    hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
    hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
    hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java

Modified: hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (original)
+++ hive/branches/branch-0.9/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java Tue Dec
 4 22:12:27 2012
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.serde2.Col
 import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
 import org.apache.hadoop.hive.serde2.columnar.BytesRefWritable;
 import org.apache.hadoop.hive.serde2.columnar.LazyDecompressionCallback;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile.Metadata;
@@ -721,7 +722,8 @@ public class RCFile {
     public Writer(FileSystem fs, Configuration conf, Path name,
         Progressable progress, Metadata metadata, CompressionCodec codec) throws IOException
{
       this(fs, conf, name, fs.getConf().getInt("io.file.buffer.size", 4096),
-          fs.getDefaultReplication(), fs.getDefaultBlockSize(), progress,
+              ShimLoader.getHadoopShims().getDefaultReplication(fs, name),
+              ShimLoader.getHadoopShims().getDefaultBlockSize(fs, name), progress,
           metadata, codec);
     }
 

Modified: hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
(original)
+++ hive/branches/branch-0.9/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
Tue Dec  4 22:12:27 2012
@@ -589,6 +589,16 @@ public class Hadoop20Shims implements Ha
   }
 
   @Override
+  public long getDefaultBlockSize(FileSystem fs, Path path) {
+    return fs.getDefaultBlockSize();
+  }
+
+  @Override
+  public short getDefaultReplication(FileSystem fs, Path path) {
+    return fs.getDefaultReplication();
+  }
+
+  @Override
   public void closeAllForUGI(UserGroupInformation ugi) {
     // No such functionality in ancient hadoop
     return;

Modified: hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
(original)
+++ hive/branches/branch-0.9/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Tue Dec  4 22:12:27 2012
@@ -21,6 +21,8 @@ import java.net.MalformedURLException;
 import java.net.URL;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.shims.HadoopShimsSecure;
 import org.apache.hadoop.mapred.ClusterStatus;
 import org.apache.hadoop.mapreduce.Job;
@@ -93,4 +95,14 @@ public class Hadoop20SShims extends Hado
   public String getJobLauncherHttpAddress(Configuration conf) {
     return conf.get("mapred.job.tracker.http.address");
   }
+
+  @Override
+  public long getDefaultBlockSize(FileSystem fs, Path path) {
+    return fs.getDefaultBlockSize();
+  }
+
+  @Override
+  public short getDefaultReplication(FileSystem fs, Path path) {
+    return fs.getDefaultReplication();
+  }
 }

Modified: hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
(original)
+++ hive/branches/branch-0.9/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
Tue Dec  4 22:12:27 2012
@@ -21,6 +21,8 @@ import java.net.MalformedURLException;
 import java.net.URL;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.shims.HadoopShims.JobTrackerState;
 import org.apache.hadoop.mapred.ClusterStatus;
 import org.apache.hadoop.mapred.JobConf;
@@ -111,4 +113,14 @@ public class Hadoop23Shims extends Hadoo
     return conf.get("yarn.resourcemanager.webapp.address");
   }
 
+  @Override
+  public long getDefaultBlockSize(FileSystem fs, Path path) {
+    return fs.getDefaultBlockSize(path);
+  }
+
+  @Override
+  public short getDefaultReplication(FileSystem fs, Path path) {
+    return fs.getDefaultReplication(path);
+  }
+
 }

Modified: hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
(original)
+++ hive/branches/branch-0.9/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
Tue Dec  4 22:12:27 2012
@@ -553,4 +553,10 @@ public abstract class HadoopShimsSecure 
 
   @Override
   abstract public String getJobLauncherRpcAddress(Configuration conf);
+
+  @Override
+  abstract public short getDefaultReplication(FileSystem fs, Path path);
+
+  @Override
+  abstract public long getDefaultBlockSize(FileSystem fs, Path path);
 }

Modified: hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1417220&r1=1417219&r2=1417220&view=diff
==============================================================================
--- hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
(original)
+++ hive/branches/branch-0.9/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Tue Dec  4 22:12:27 2012
@@ -277,6 +277,24 @@ public interface HadoopShims {
   public String getJobLauncherHttpAddress(Configuration conf);
 
   /**
+   * Get the default block size for the path. FileSystem alone is not sufficient to
+   * determine the same, as in case of CSMT the underlying file system determines that.
+   * @param fs
+   * @param path
+   * @return
+   */
+  public long getDefaultBlockSize(FileSystem fs, Path path);
+
+  /**
+   * Get the default replication for a path. In case of CSMT the given path will be used
to
+   * locate the actual filesystem.
+   * @param fs
+   * @param path
+   * @return
+   */
+  public short getDefaultReplication(FileSystem fs, Path path);
+
+  /**
    * InputSplitShim.
    *
    */



Mime
View raw message