hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject [75/91] [abbrv] hive git commit: HIVE-12498: ACID: Setting OrcRecordUpdater.OrcOptions.tableProperties() has no effect (Prasanth Jayachandran reviewed by Eugene Koifman)
Date Mon, 30 Nov 2015 00:12:12 GMT
HIVE-12498: ACID: Setting OrcRecordUpdater.OrcOptions.tableProperties() has no effect (Prasanth
Jayachandran reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3cb23b9e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3cb23b9e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3cb23b9e

Branch: refs/heads/spark
Commit: 3cb23b9e4b0a0e2a8edc3c2f5ee040e5ad01142d
Parents: 4201179
Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Authored: Wed Nov 25 12:10:02 2015 -0600
Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Committed: Wed Nov 25 12:10:02 2015 -0600

----------------------------------------------------------------------
 .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java |  3 +-
 .../hive/ql/io/orc/TestOrcRecordUpdater.java    | 58 ++++++++++++++++++--
 2 files changed, 54 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/3cb23b9e/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 67c5a11..ee31c23 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -247,7 +247,8 @@ public class OrcRecordUpdater implements RecordUpdater {
       writerOptions = ((OrcOptions) options).getOrcOptions();
     }
     if (writerOptions == null) {
-      writerOptions = OrcFile.writerOptions(options.getConfiguration());
+      writerOptions = OrcFile.writerOptions(options.getTableProperties(),
+          options.getConfiguration());
     }
     writerOptions.fileSystem(fs).callback(indexBuilder);
     if (!options.isWritingBase()) {

http://git-wip-us.apache.org/repos/asf/hive/blob/3cb23b9e/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
index 22030b4..973cc40 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java
@@ -18,6 +18,15 @@
 
 package org.apache.hadoop.hive.ql.io.orc;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.util.Properties;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -33,12 +42,6 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.Reporter;
 import org.junit.Test;
 
-import java.io.DataInputStream;
-import java.io.File;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
 public class TestOrcRecordUpdater {
 
   @Test
@@ -180,6 +183,49 @@ public class TestOrcRecordUpdater {
   }
 
   @Test
+  public void testWriterTblProperties() throws Exception {
+    Path root = new Path(workDir, "testWriterTblProperties");
+    Configuration conf = new Configuration();
+    // Must use raw local because the checksummer doesn't honor flushes.
+    FileSystem fs = FileSystem.getLocal(conf).getRaw();
+    ObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = ObjectInspectorFactory.getReflectionObjectInspector
+          (MyRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+    Properties tblProps = new Properties();
+    tblProps.setProperty("orc.compress", "SNAPPY");
+    AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
+        .filesystem(fs)
+        .bucket(10)
+        .writingBase(false)
+        .minimumTransactionId(10)
+        .maximumTransactionId(19)
+        .inspector(inspector)
+        .reporter(Reporter.NULL)
+        .finalDestination(root)
+        .tableProperties(tblProps);
+    RecordUpdater updater = new OrcRecordUpdater(root, options);
+    updater.insert(11, new MyRow("first"));
+    updater.insert(11, new MyRow("second"));
+    updater.insert(11, new MyRow("third"));
+    updater.flush();
+    updater.insert(12, new MyRow("fourth"));
+    updater.insert(12, new MyRow("fifth"));
+    updater.flush();
+
+    PrintStream origOut = System.out;
+    ByteArrayOutputStream myOut = new ByteArrayOutputStream();
+    System.setOut(new PrintStream(myOut));
+    FileDump.main(new String[]{root.toUri().toString()});
+    System.out.flush();
+    String outDump = new String(myOut.toByteArray());
+    assertEquals(true, outDump.contains("Compression: SNAPPY"));
+    System.setOut(origOut);
+    updater.close(false);
+  }
+
+  @Test
   public void testUpdates() throws Exception {
     Path root = new Path(workDir, "testUpdates");
     Configuration conf = new Configuration();


Mime
View raw message