hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1480526 [2/2] - in /hive/branches/vectorization: ./ data/files/ hcatalog/build-support/ant/ hcatalog/server-extensions/src/main/java/org/apache/hcatalog/listener/ hcatalog/src/test/ hcatalog/src/test/e2e/templeton/drivers/ hcatalog/src/tes...
Date Thu, 09 May 2013 04:38:39 GMT
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
(original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
Thu May  9 04:38:38 2013
@@ -52,6 +52,7 @@ import org.junit.Test;
 import org.junit.rules.TestName;
 
 import java.io.File;
+import java.io.IOException;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.sql.Timestamp;
@@ -639,7 +640,7 @@ public class TestOrcFile {
     row.setFieldValue(0, null);
     union.set((byte) 0, new IntWritable(1732050807));
     row.setFieldValue(2, null);
-    for(int i=0; i < 1000; ++i) {
+    for(int i=0; i < 5000; ++i) {
       writer.addRow(row);
     }
     union.set((byte) 0, new IntWritable(0));
@@ -651,7 +652,7 @@ public class TestOrcFile {
     writer.close();
     Reader reader = OrcFile.createReader(fs, testFilePath);
     assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
-    assertEquals(1309, reader.getNumberOfRows());
+    assertEquals(5309, reader.getNumberOfRows());
     DecimalColumnStatistics stats =
         (DecimalColumnStatistics) reader.getStatistics()[5];
     assertEquals(303, stats.getNumberOfValues());
@@ -732,7 +733,7 @@ public class TestOrcFile {
       assertEquals(new HiveDecimal(new BigInteger(118, rand),
                                    rand.nextInt(36)), row.getFieldValue(2));
     }
-    for(int i=0; i < 1000; ++i) {
+    for(int i=0; i < 5000; ++i) {
       row = (OrcStruct) rows.next(row);
       assertEquals(new IntWritable(1732050807), union.getObject());
     }
@@ -942,6 +943,8 @@ public class TestOrcFile {
     double rate;
     Path path = null;
     long lastAllocation = 0;
+    int rows = 0;
+    MemoryManager.Callback callback;
 
     MyMemoryManager(Configuration conf, long totalSpace, double rate) {
       super(conf);
@@ -954,6 +957,7 @@ public class TestOrcFile {
                    MemoryManager.Callback callback) {
       this.path = path;
       this.lastAllocation = requestedAllocation;
+      this.callback = callback;
     }
 
     @Override
@@ -971,6 +975,13 @@ public class TestOrcFile {
     double getAllocationScale() {
       return rate;
     }
+
+    @Override
+    void addedRow() throws IOException {
+      if (++rows % 100 == 0) {
+        callback.checkMemory(rate);
+      }
+    }
   }
 
   @Test
@@ -995,9 +1006,9 @@ public class TestOrcFile {
     for(StripeInformation stripe: reader.getStripes()) {
       i += 1;
       assertTrue("stripe " + i + " is too long at " + stripe.getDataLength(),
-          stripe.getDataLength() < 10000);
+          stripe.getDataLength() < 5000);
     }
-    assertEquals(3, i);
+    assertEquals(25, i);
     assertEquals(2500, reader.getNumberOfRows());
   }
 }

Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
(original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
Thu May  9 04:38:38 2013
@@ -104,7 +104,7 @@ public class TestStringRedBlackTree {
       System.err.println(indent + "NULL");
     } else {
       System.err.println(indent + "Node " + node + " color " +
-        (tree.isRed(node) ? "red" : "black") + " count " + tree.getCount(node));
+        (tree.isRed(node) ? "red" : "black"));
       printTree(tree, indent + "  ", tree.getLeft(node));
       printTree(tree, indent + "  ", tree.getRight(node));
     }
@@ -112,14 +112,12 @@ public class TestStringRedBlackTree {
 
   private static class MyVisitor implements StringRedBlackTree.Visitor {
     private final String[] words;
-    private final int[] counts;
     private final int[] order;
     private final DataOutputBuffer buffer = new DataOutputBuffer();
     int current = 0;
 
-    MyVisitor(String[] args, int[] counts, int[] order) {
+    MyVisitor(String[] args, int[] order) {
       words = args;
-      this.counts = counts;
       this.order = order;
     }
 
@@ -128,7 +126,6 @@ public class TestStringRedBlackTree {
                      ) throws IOException {
       String word = context.getText().toString();
       assertEquals("in word " + current, words[current], word);
-      assertEquals("in word " + current, counts[current], context.getCount());
       assertEquals("in word " + current, order[current],
         context.getOriginalPosition());
       buffer.reset();
@@ -138,14 +135,14 @@ public class TestStringRedBlackTree {
     }
   }
 
-  void checkContents(StringRedBlackTree tree, int[] counts, int[] order,
+  void checkContents(StringRedBlackTree tree, int[] order,
                      String... params
                     ) throws IOException {
-    tree.visit(new MyVisitor(params, counts, order));
+    tree.visit(new MyVisitor(params, order));
   }
 
   StringRedBlackTree buildTree(String... params) throws IOException {
-    StringRedBlackTree result = new StringRedBlackTree();
+    StringRedBlackTree result = new StringRedBlackTree(1000);
     for(String word: params) {
       result.add(word);
       checkTree(result);
@@ -156,7 +153,7 @@ public class TestStringRedBlackTree {
   @Test
   public void test1() throws Exception {
     StringRedBlackTree tree = new StringRedBlackTree(5);
-    assertEquals(0, tree.getByteSize());
+    assertEquals(0, tree.getSizeInBytes());
     checkTree(tree);
     assertEquals(0, tree.add("owen"));
     checkTree(tree);
@@ -186,15 +183,12 @@ public class TestStringRedBlackTree {
     checkTree(tree);
     assertEquals(9, tree.add("z"));
     checkTree(tree);
-    checkContents(tree, new int[]{2,1,2,1,1,1,1,2,1,1},
-      new int[]{2,5,1,4,6,3,7,0,9,8},
+    checkContents(tree, new int[]{2,5,1,4,6,3,7,0,9,8},
       "alan", "arun", "ashutosh", "eric", "eric14", "greg",
       "o", "owen", "z", "ziggy");
-    assertEquals(10*5*4 + 8 + 6 + 5 + 5 * 4 + 2 * 1, tree.getByteSize());
+    assertEquals(32888, tree.getSizeInBytes());
     // check that adding greg again bumps the count
-    assertEquals(1, tree.getCount(3));
     assertEquals(3, tree.add("greg"));
-    assertEquals(2, tree.getCount(3));
     assertEquals(41, tree.getCharacterSize());
     // add some more strings to test the different branches of the
     // rebalancing
@@ -210,7 +204,7 @@ public class TestStringRedBlackTree {
     checkTree(tree);
     tree.clear();
     checkTree(tree);
-    assertEquals(0, tree.getByteSize());
+    assertEquals(0, tree.getSizeInBytes());
     assertEquals(0, tree.getCharacterSize());
   }
 
@@ -220,8 +214,7 @@ public class TestStringRedBlackTree {
       buildTree("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
         "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
     assertEquals(26, tree.size());
-    checkContents(tree, new int[]{1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1,
-      1,1,1, 1,1,1, 1,1}, new int[]{0,1,2, 3,4,5, 6,7,8, 9,10,11, 12,13,14,
+    checkContents(tree, new int[]{0,1,2, 3,4,5, 6,7,8, 9,10,11, 12,13,14,
       15,16,17, 18,19,20, 21,22,23, 24,25},
       "a", "b", "c", "d", "e", "f", "g", "h", "i", "j","k", "l", "m", "n", "o",
       "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
@@ -233,8 +226,7 @@ public class TestStringRedBlackTree {
       buildTree("z", "y", "x", "w", "v", "u", "t", "s", "r", "q", "p", "o", "n",
         "m", "l", "k", "j", "i", "h", "g", "f", "e", "d", "c", "b", "a");
     assertEquals(26, tree.size());
-    checkContents(tree, new int[]{1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1, 1,1,1,
-      1,1,1, 1,1,1, 1,1}, new int[]{25,24,23, 22,21,20, 19,18,17, 16,15,14,
+    checkContents(tree, new int[]{25,24,23, 22,21,20, 19,18,17, 16,15,14,
       13,12,11, 10,9,8, 7,6,5, 4,3,2, 1,0},
       "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
       "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");

Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/column_access_stats.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/column_access_stats.q?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/column_access_stats.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/column_access_stats.q Thu
May  9 04:38:38 2013
@@ -8,11 +8,15 @@ LOAD DATA LOCAL INPATH '../data/files/T1
 
 CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE;
 CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE;
+CREATE TABLE T4(key STRING, val STRING) PARTITIONED BY (p STRING);
 
 -- Simple select queries
 SELECT key FROM T1 ORDER BY key;
 SELECT key, val FROM T1 ORDER BY key, val;
 SELECT 1 FROM T1;
+SELECT key, val from T4 where p=1;
+SELECT val FROM T4 where p=1;
+SELECT p, val FROM T4 where p=1;
 
 -- More complicated select queries
 EXPLAIN SELECT key FROM (SELECT key, val FROM T1) subq1 ORDER BY key;

Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/orc_create.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/orc_create.q?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/orc_create.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/orc_create.q Thu May  9
04:38:38 2013
@@ -1,4 +1,18 @@
 DROP TABLE orc_create;
+DROP TABLE orc_create_complex;
+DROP TABLE orc_create_staging;
+
+CREATE TABLE orc_create_staging (
+  str STRING,
+  mp  MAP<STRING,STRING>,
+  lst ARRAY<STRING>,
+  strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY '|'
+    COLLECTION ITEMS TERMINATED BY ','
+    MAP KEYS TERMINATED BY ':';
+
+DESCRIBE FORMATTED orc_create_staging;
 
 CREATE TABLE orc_create (key INT, value STRING)
    PARTITIONED BY (ds string)
@@ -26,4 +40,27 @@ CREATE TABLE orc_create (key INT, value 
 
 DESCRIBE FORMATTED orc_create;
 
+CREATE TABLE orc_create_complex (
+  str STRING,
+  mp  MAP<STRING,STRING>,
+  lst ARRAY<STRING>,
+  strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC;
+
+DESCRIBE FORMATTED orc_create_complex;
+
+LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging;
+
+SELECT * from orc_create_staging;
+
+INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging;
+
+SELECT * from orc_create_complex;
+SELECT str from orc_create_complex;
+SELECT mp from orc_create_complex;
+SELECT lst from orc_create_complex;
+SELECT strct from orc_create_complex;
+
 DROP TABLE orc_create;
+DROP TABLE orc_create_complex;
+DROP TABLE orc_create_staging;

Modified: hive/branches/vectorization/ql/src/test/resources/orc-file-dump.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/resources/orc-file-dump.out?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/resources/orc-file-dump.out (original)
+++ hive/branches/vectorization/ql/src/test/resources/orc-file-dump.out Thu May  9 04:38:38
2013
@@ -11,74 +11,87 @@ Statistics:
   Column 3: count: 21000 min: Darkness, max: worst
 
 Stripes:
-  Stripe: offset: 3 data: 83505 rows: 6000 tail: 91 index: 179
+  Stripe: offset: 3 data: 69638 rows: 5000 tail: 85 index: 126
     Stream: column 0 section ROW_INDEX start: 3 length 10
     Stream: column 1 section ROW_INDEX start: 13 length 38
     Stream: column 2 section ROW_INDEX start: 51 length 42
-    Stream: column 3 section DICTIONARY_COUNT start: 93 length 53
-    Stream: column 3 section ROW_INDEX start: 146 length 36
-    Stream: column 1 section PRESENT start: 182 length 11
-    Stream: column 1 section DATA start: 193 length 27086
-    Stream: column 2 section PRESENT start: 27279 length 11
-    Stream: column 2 section DATA start: 27290 length 52124
-    Stream: column 3 section PRESENT start: 79414 length 11
-    Stream: column 3 section DATA start: 79425 length 4091
-    Stream: column 3 section LENGTH start: 83516 length 38
-    Stream: column 3 section DICTIONARY_DATA start: 83554 length 133
+    Stream: column 3 section ROW_INDEX start: 93 length 36
+    Stream: column 1 section PRESENT start: 129 length 11
+    Stream: column 1 section DATA start: 140 length 22605
+    Stream: column 2 section PRESENT start: 22745 length 11
+    Stream: column 2 section DATA start: 22756 length 43426
+    Stream: column 3 section PRESENT start: 66182 length 11
+    Stream: column 3 section DATA start: 66193 length 3403
+    Stream: column 3 section LENGTH start: 69596 length 38
+    Stream: column 3 section DICTIONARY_DATA start: 69634 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT
     Encoding column 2: DIRECT
     Encoding column 3: DICTIONARY[35]
-  Stripe: offset: 83778 data: 83453 rows: 6000 tail: 91 index: 180
-    Stream: column 0 section ROW_INDEX start: 83778 length 10
-    Stream: column 1 section ROW_INDEX start: 83788 length 39
-    Stream: column 2 section ROW_INDEX start: 83827 length 42
-    Stream: column 3 section DICTIONARY_COUNT start: 83869 length 53
-    Stream: column 3 section ROW_INDEX start: 83922 length 36
-    Stream: column 1 section PRESENT start: 83958 length 11
-    Stream: column 1 section DATA start: 83969 length 27093
-    Stream: column 2 section PRESENT start: 111062 length 11
-    Stream: column 2 section DATA start: 111073 length 52119
-    Stream: column 3 section PRESENT start: 163192 length 11
-    Stream: column 3 section DATA start: 163203 length 4037
-    Stream: column 3 section LENGTH start: 167240 length 38
-    Stream: column 3 section DICTIONARY_DATA start: 167278 length 133
+  Stripe: offset: 69852 data: 69617 rows: 5000 tail: 83 index: 124
+    Stream: column 0 section ROW_INDEX start: 69852 length 10
+    Stream: column 1 section ROW_INDEX start: 69862 length 36
+    Stream: column 2 section ROW_INDEX start: 69898 length 42
+    Stream: column 3 section ROW_INDEX start: 69940 length 36
+    Stream: column 1 section PRESENT start: 69976 length 11
+    Stream: column 1 section DATA start: 69987 length 22597
+    Stream: column 2 section PRESENT start: 92584 length 11
+    Stream: column 2 section DATA start: 92595 length 43439
+    Stream: column 3 section PRESENT start: 136034 length 11
+    Stream: column 3 section DATA start: 136045 length 3377
+    Stream: column 3 section LENGTH start: 139422 length 38
+    Stream: column 3 section DICTIONARY_DATA start: 139460 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT
     Encoding column 2: DIRECT
     Encoding column 3: DICTIONARY[35]
-  Stripe: offset: 167502 data: 83456 rows: 6000 tail: 92 index: 182
-    Stream: column 0 section ROW_INDEX start: 167502 length 10
-    Stream: column 1 section ROW_INDEX start: 167512 length 39
-    Stream: column 2 section ROW_INDEX start: 167551 length 42
-    Stream: column 3 section DICTIONARY_COUNT start: 167593 length 55
-    Stream: column 3 section ROW_INDEX start: 167648 length 36
-    Stream: column 1 section PRESENT start: 167684 length 11
-    Stream: column 1 section DATA start: 167695 length 27080
-    Stream: column 2 section PRESENT start: 194775 length 11
-    Stream: column 2 section DATA start: 194786 length 52093
-    Stream: column 3 section PRESENT start: 246879 length 11
-    Stream: column 3 section DATA start: 246890 length 4079
-    Stream: column 3 section LENGTH start: 250969 length 38
-    Stream: column 3 section DICTIONARY_DATA start: 251007 length 133
+  Stripe: offset: 139676 data: 69603 rows: 5000 tail: 85 index: 127
+    Stream: column 0 section ROW_INDEX start: 139676 length 10
+    Stream: column 1 section ROW_INDEX start: 139686 length 39
+    Stream: column 2 section ROW_INDEX start: 139725 length 42
+    Stream: column 3 section ROW_INDEX start: 139767 length 36
+    Stream: column 1 section PRESENT start: 139803 length 11
+    Stream: column 1 section DATA start: 139814 length 22594
+    Stream: column 2 section PRESENT start: 162408 length 11
+    Stream: column 2 section DATA start: 162419 length 43415
+    Stream: column 3 section PRESENT start: 205834 length 11
+    Stream: column 3 section DATA start: 205845 length 3390
+    Stream: column 3 section LENGTH start: 209235 length 38
+    Stream: column 3 section DICTIONARY_DATA start: 209273 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT
     Encoding column 2: DIRECT
     Encoding column 3: DICTIONARY[35]
-  Stripe: offset: 251232 data: 41842 rows: 3000 tail: 90 index: 172
-    Stream: column 0 section ROW_INDEX start: 251232 length 10
-    Stream: column 1 section ROW_INDEX start: 251242 length 39
-    Stream: column 2 section ROW_INDEX start: 251281 length 43
-    Stream: column 3 section DICTIONARY_COUNT start: 251324 length 44
-    Stream: column 3 section ROW_INDEX start: 251368 length 36
-    Stream: column 1 section PRESENT start: 251404 length 9
-    Stream: column 1 section DATA start: 251413 length 13544
-    Stream: column 2 section PRESENT start: 264957 length 9
-    Stream: column 2 section DATA start: 264966 length 26072
-    Stream: column 3 section PRESENT start: 291038 length 9
-    Stream: column 3 section DATA start: 291047 length 2028
-    Stream: column 3 section LENGTH start: 293075 length 38
-    Stream: column 3 section DICTIONARY_DATA start: 293113 length 133
+  Stripe: offset: 209491 data: 69584 rows: 5000 tail: 84 index: 126
+    Stream: column 0 section ROW_INDEX start: 209491 length 10
+    Stream: column 1 section ROW_INDEX start: 209501 length 38
+    Stream: column 2 section ROW_INDEX start: 209539 length 42
+    Stream: column 3 section ROW_INDEX start: 209581 length 36
+    Stream: column 1 section PRESENT start: 209617 length 11
+    Stream: column 1 section DATA start: 209628 length 22575
+    Stream: column 2 section PRESENT start: 232203 length 11
+    Stream: column 2 section DATA start: 232214 length 43426
+    Stream: column 3 section PRESENT start: 275640 length 11
+    Stream: column 3 section DATA start: 275651 length 3379
+    Stream: column 3 section LENGTH start: 279030 length 38
+    Stream: column 3 section DICTIONARY_DATA start: 279068 length 133
+    Encoding column 0: DIRECT
+    Encoding column 1: DIRECT
+    Encoding column 2: DIRECT
+    Encoding column 3: DICTIONARY[35]
+  Stripe: offset: 279285 data: 14111 rows: 1000 tail: 80 index: 127
+    Stream: column 0 section ROW_INDEX start: 279285 length 10
+    Stream: column 1 section ROW_INDEX start: 279295 length 39
+    Stream: column 2 section ROW_INDEX start: 279334 length 42
+    Stream: column 3 section ROW_INDEX start: 279376 length 36
+    Stream: column 1 section PRESENT start: 279412 length 5
+    Stream: column 1 section DATA start: 279417 length 4529
+    Stream: column 2 section PRESENT start: 283946 length 5
+    Stream: column 2 section DATA start: 283951 length 8690
+    Stream: column 3 section PRESENT start: 292641 length 5
+    Stream: column 3 section DATA start: 292646 length 706
+    Stream: column 3 section LENGTH start: 293352 length 38
+    Stream: column 3 section DICTIONARY_DATA start: 293390 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT
     Encoding column 2: DIRECT

Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out
(original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/column_access_stats.q.out
Thu May  9 04:38:38 2013
@@ -9,6 +9,8 @@ PREHOOK: query: CREATE TABLE T2(key STRI
 PREHOOK: type: CREATETABLE
 PREHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE
 PREHOOK: type: CREATETABLE
+PREHOOK: query: CREATE TABLE T4(key STRING, val STRING) PARTITIONED BY (p STRING)
+PREHOOK: type: CREATETABLE
 PREHOOK: query: -- Simple select queries
 SELECT key FROM T1 ORDER BY key
 PREHOOK: type: QUERY
@@ -46,6 +48,27 @@ PREHOOK: Input: default@t1
 1
 1
 1
+PREHOOK: query: SELECT key, val from T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:key,val
+
+PREHOOK: query: SELECT val FROM T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:val
+
+PREHOOK: query: SELECT p, val FROM T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:val
+
 PREHOOK: query: -- More complicated select queries
 EXPLAIN SELECT key FROM (SELECT key, val FROM T1) subq1 ORDER BY key
 PREHOOK: type: QUERY

Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/orc_create.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/orc_create.q.out?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/orc_create.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/orc_create.q.out Thu May
 9 04:38:38 2013
@@ -2,6 +2,69 @@ PREHOOK: query: DROP TABLE orc_create
 PREHOOK: type: DROPTABLE
 POSTHOOK: query: DROP TABLE orc_create
 POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_create_complex
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_create_complex
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_create_staging
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_create_staging
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE orc_create_staging (
+  str STRING,
+  mp  MAP<STRING,STRING>,
+  lst ARRAY<STRING>,
+  strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY '|'
+    COLLECTION ITEMS TERMINATED BY ','
+    MAP KEYS TERMINATED BY ':'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE orc_create_staging (
+  str STRING,
+  mp  MAP<STRING,STRING>,
+  lst ARRAY<STRING>,
+  strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY '|'
+    COLLECTION ITEMS TERMINATED BY ','
+    MAP KEYS TERMINATED BY ':'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_create_staging
+PREHOOK: query: DESCRIBE FORMATTED orc_create_staging
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED orc_create_staging
+POSTHOOK: type: DESCTABLE
+# col_name            	data_type           	comment             
+	 	 
+str                 	string              	None                
+mp                  	map<string,string>  	None                
+lst                 	array<string>       	None                
+strct               	struct<A:string,B:string>	None                
+	 	 
+# Detailed Table Information	 	 
+Database:           	default             	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+#### A masked pattern was here ####
+Table Type:         	MANAGED_TABLE       	 
+Table Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	colelction.delim    	,                   
+	field.delim         	|                   
+	mapkey.delim        	:                   
+	serialization.format	|                   
 PREHOOK: query: CREATE TABLE orc_create (key INT, value STRING)
    PARTITIONED BY (ds string)
    STORED AS ORC
@@ -185,6 +248,156 @@ Bucket Columns:     	[]                 
 Sort Columns:       	[]                  	 
 Storage Desc Params:	 	 
 	serialization.format	1                   
+PREHOOK: query: CREATE TABLE orc_create_complex (
+  str STRING,
+  mp  MAP<STRING,STRING>,
+  lst ARRAY<STRING>,
+  strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE orc_create_complex (
+  str STRING,
+  mp  MAP<STRING,STRING>,
+  lst ARRAY<STRING>,
+  strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_create_complex
+PREHOOK: query: DESCRIBE FORMATTED orc_create_complex
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED orc_create_complex
+POSTHOOK: type: DESCTABLE
+# col_name            	data_type           	comment             
+	 	 
+str                 	string              	from deserializer   
+mp                  	map<string,string>  	from deserializer   
+lst                 	array<string>       	from deserializer   
+strct               	struct<A:string,B:string>	from deserializer   
+	 	 
+# Detailed Table Information	 	 
+Database:           	default             	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+#### A masked pattern was here ####
+Table Type:         	MANAGED_TABLE       	 
+Table Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.ql.io.orc.OrcSerde	 
+InputFormat:        	org.apache.hadoop.hive.ql.io.orc.OrcInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE
orc_create_staging
+PREHOOK: type: LOAD
+PREHOOK: Output: default@orc_create_staging
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE
orc_create_staging
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@orc_create_staging
+PREHOOK: query: SELECT * from orc_create_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_staging
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from orc_create_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_staging
+#### A masked pattern was here ####
+line1	{"key11":"value11","key12":"value12","key13":"value13"}	["a","b","c"]	{"a":"one","b":"two"}
+line2	{"key21":"value21","key22":"value22","key23":"value23"}	["d","e","f"]	{"a":"three","b":"four"}
+line3	{"key31":"value31","key32":"value32","key33":"value33"}	["g","h","i"]	{"a":"five","b":"six"}
+PREHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_staging
+PREHOOK: Output: default@orc_create_complex
+POSTHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_staging
+POSTHOOK: Output: default@orc_create_complex
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: SELECT * from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]
+line1	{"key12":"value12","key11":"value11","key13":"value13"}	["a","b","c"]	{"A":"one","B":"two"}
+line2	{"key21":"value21","key23":"value23","key22":"value22"}	["d","e","f"]	{"A":"three","B":"four"}
+line3	{"key33":"value33","key31":"value31","key32":"value32"}	["g","h","i"]	{"A":"five","B":"six"}
+PREHOOK: query: SELECT str from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT str from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]
+line1
+line2
+line3
+PREHOOK: query: SELECT mp from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT mp from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]
+{"key12":"value12","key11":"value11","key13":"value13"}
+{"key21":"value21","key23":"value23","key22":"value22"}
+{"key33":"value33","key31":"value31","key32":"value32"}
+PREHOOK: query: SELECT lst from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT lst from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]
+["a","b","c"]
+["d","e","f"]
+["g","h","i"]
+PREHOOK: query: SELECT strct from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT strct from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]
+{"a":"one","b":"two"}
+{"a":"three","b":"four"}
+{"a":"five","b":"six"}
 PREHOOK: query: DROP TABLE orc_create
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@orc_create
@@ -193,3 +406,31 @@ POSTHOOK: query: DROP TABLE orc_create
 POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@orc_create
 POSTHOOK: Output: default@orc_create
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: DROP TABLE orc_create_complex
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_complex
+PREHOOK: Output: default@orc_create_complex
+POSTHOOK: query: DROP TABLE orc_create_complex
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_complex
+POSTHOOK: Output: default@orc_create_complex
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: DROP TABLE orc_create_staging
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_staging
+PREHOOK: Output: default@orc_create_staging
+POSTHOOK: query: DROP TABLE orc_create_staging
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_staging
+POSTHOOK: Output: default@orc_create_staging
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst,
type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp,
type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str,
type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct,
type:struct<A:string,B:string>, comment:null), ]

Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
(original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
Thu May  9 04:38:38 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.ql.processors.CommandProcessor;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hive.service.cli.FetchOrientation;
 import org.apache.hive.service.cli.HiveSQLException;
 import org.apache.hive.service.cli.OperationState;
@@ -85,6 +86,12 @@ public abstract class HiveCommandOperati
     }
   }
 
+
+  private void tearDownSessionIO() {
+    IOUtils.cleanup(LOG, parentSession.getSessionState().out);
+    IOUtils.cleanup(LOG, parentSession.getSessionState().err);
+  }
+
   /* (non-Javadoc)
    * @see org.apache.hive.service.cli.operation.Operation#run()
    */
@@ -121,6 +128,7 @@ public abstract class HiveCommandOperati
   @Override
   public void close() throws HiveSQLException {
     setState(OperationState.CLOSED);
+    tearDownSessionIO();
     cleanTmpFile();
   }
 
@@ -187,6 +195,7 @@ public abstract class HiveCommandOperati
     if (resultReader != null) {
       SessionState sessionState = getParentSession().getSessionState();
       File tmp = sessionState.getTmpOutputFile();
+      IOUtils.cleanup(LOG, resultReader);
       tmp.delete();
       resultReader = null;
     }

Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/Operation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/Operation.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/Operation.java
(original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/Operation.java
Thu May  9 04:38:38 2013
@@ -32,7 +32,7 @@ import org.apache.hive.service.cli.sessi
 
 
 public abstract class Operation {
-  private final HiveSession parentSession;
+  protected final HiveSession parentSession;
   private OperationState state = OperationState.INITIALIZED;
   private final OperationHandle opHandle;
   private HiveConf configuration;

Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1480526&r1=1480525&r2=1480526&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
(original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
Thu May  9 04:38:38 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.history.HiveHistory;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hive.common.util.HiveVersionInfo;
 import org.apache.hive.service.cli.FetchOrientation;
@@ -295,6 +296,10 @@ public class HiveSessionImpl implements 
         operationManager.closeOperation(opHandle);
       }
       opHandleSet.clear();
+      HiveHistory hiveHist = sessionState.getHiveHistory();
+      if (null != hiveHist) {
+        hiveHist.closeStream();
+      }
     } finally {
       release();
     }
@@ -367,4 +372,4 @@ public class HiveSessionImpl implements 
   protected HiveSession getSession() {
     return this;
   }
-}
\ No newline at end of file
+}



Mime
View raw message