hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1599020 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: exec/ exec/persistence/ io/
Date Sun, 01 Jun 2014 17:56:45 GMT
Author: hashutosh
Date: Sun Jun  1 17:56:44 2014
New Revision: 1599020

URL: http://svn.apache.org/r1599020
Log:
HIVE-7137 : Add progressable to writer interfaces so they could report progress while different
operations are in progress (Sumit Kumar via Ashutosh Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Sun Jun  1 17:56:44
2014
@@ -177,6 +177,7 @@ import org.apache.hadoop.mapred.RecordRe
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Shell;
 
@@ -1354,9 +1355,9 @@ public final class Utilities {
    * @return output stream over the created sequencefile
    */
   public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path
file,
-      Class<?> keyClass, Class<?> valClass) throws IOException {
+      Class<?> keyClass, Class<?> valClass, Progressable progressable) throws
IOException {
     boolean isCompressed = FileOutputFormat.getCompressOutput(jc);
-    return createSequenceWriter(jc, fs, file, keyClass, valClass, isCompressed);
+    return createSequenceWriter(jc, fs, file, keyClass, valClass, isCompressed, progressable);
   }
 
   /**
@@ -1376,7 +1377,8 @@ public final class Utilities {
    * @return output stream over the created sequencefile
    */
   public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path
file,
-      Class<?> keyClass, Class<?> valClass, boolean isCompressed) throws IOException
{
+      Class<?> keyClass, Class<?> valClass, boolean isCompressed, Progressable
progressable)
+      throws IOException {
     CompressionCodec codec = null;
     CompressionType compressionType = CompressionType.NONE;
     Class codecClass = null;
@@ -1385,7 +1387,8 @@ public final class Utilities {
       codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
       codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
     }
-    return (SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType,
codec));
+    return (SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType,
codec,
+	progressable));
 
   }
 
@@ -1402,14 +1405,14 @@ public final class Utilities {
    * @return output stream over the created rcfile
    */
   public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file,
-      boolean isCompressed) throws IOException {
+      boolean isCompressed, Progressable progressable) throws IOException {
     CompressionCodec codec = null;
     Class<?> codecClass = null;
     if (isCompressed) {
       codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
       codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
     }
-    return new RCFile.Writer(fs, jc, file, null, codec);
+    return new RCFile.Writer(fs, jc, file, progressable, codec);
   }
 
   /**

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
Sun Jun  1 17:56:44 2014
@@ -269,8 +269,8 @@ public class PTFRowContainer<Row extends
         Properties tableProperties, Progressable progress) throws IOException {
 
       FileSystem fs = finalOutPath.getFileSystem(jc);
-      final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
-          fs, finalOutPath, BytesWritable.class, valueClass, isCompressed);
+      final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc, fs, finalOutPath,
+	  BytesWritable.class, valueClass, isCompressed, progress);
 
       return new PTFRecordWriter(outStream);
     }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java Sun Jun
 1 17:56:44 2014
@@ -64,7 +64,7 @@ public class HiveBinaryOutputFormat<K ex
       Properties tableProperties, Progressable progress) throws IOException {
 
     FileSystem fs = outPath.getFileSystem(jc);
-    final OutputStream outStream = fs.create(outPath);
+    final OutputStream outStream = fs.create(outPath, progress);
 
     return new RecordWriter() {
       @Override

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
Sun Jun  1 17:56:44 2014
@@ -77,8 +77,8 @@ public class HiveIgnoreKeyTextOutputForm
 
     final int finalRowSeparator = rowSeparator;
     FileSystem fs = outPath.getFileSystem(jc);
-    final OutputStream outStream = Utilities.createCompressedStream(jc, fs
-        .create(outPath), isCompressed);
+    final OutputStream outStream = Utilities.createCompressedStream(jc,
+	fs.create(outPath, progress), isCompressed);
     return new RecordWriter() {
       @Override
       public void write(Writable r) throws IOException {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
Sun Jun  1 17:56:44 2014
@@ -53,8 +53,8 @@ public class HiveNullValueSequenceFileOu
       Properties tableProperties, Progressable progress) throws IOException {
 
     FileSystem fs = finalOutPath.getFileSystem(jc);
-    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
-        fs, finalOutPath, HiveKey.class, NullWritable.class, isCompressed);
+    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc, fs, finalOutPath,
+	HiveKey.class, NullWritable.class, isCompressed, progress);
 
     keyWritable = new HiveKey();
     keyIsText = valueClass.equals(Text.class);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
Sun Jun  1 17:56:44 2014
@@ -61,8 +61,8 @@ public class HiveSequenceFileOutputForma
       Properties tableProperties, Progressable progress) throws IOException {
 
     FileSystem fs = finalOutPath.getFileSystem(jc);
-    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
-        fs, finalOutPath, BytesWritable.class, valueClass, isCompressed);
+    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc, fs, finalOutPath,
+	BytesWritable.class, valueClass, isCompressed, progress);
 
     return new RecordWriter() {
       @Override

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java Sun Jun  1
17:56:44 2014
@@ -131,9 +131,8 @@ public class RCFileOutputFormat extends
     }
 
     RCFileOutputFormat.setColumnNumber(jc, cols.length);
-    final RCFile.Writer outWriter = Utilities.createRCFileWriter
-      (jc, finalOutPath.getFileSystem(jc),
-       finalOutPath, isCompressed);
+    final RCFile.Writer outWriter = Utilities.createRCFileWriter(jc,
+	finalOutPath.getFileSystem(jc), finalOutPath, isCompressed, progress);
 
     return new org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter() {
       @Override



Mime
View raw message