hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r922404 [2/3] - in /hadoop/hive/trunk: ./ hbase-handler/ hbase-handler/lib/ hbase-handler/src/ hbase-handler/src/java/ hbase-handler/src/java/org/ hbase-handler/src/java/org/apache/ hbase-handler/src/java/org/apache/hadoop/ hbase-handler/sr...
Date Fri, 12 Mar 2010 19:53:59 GMT
Added: hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java (added)
+++ hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java Fri Mar 12 19:53:57 2010
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.hbase;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.HbaseMapWritable;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyString;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+/**
+ * TestLazyHBaseObject is a test for the LazyHBaseXXX classes.
+ */
+public class TestLazyHBaseObject extends TestCase {
+  /**
+   * Test the LazyMap class with Integer-to-String.
+   */
+  public void testLazyHBaseCellMap1() {
+    // Map of Integer to String
+    Text nullSequence = new Text("\\N");
+    ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+      TypeInfoUtils.getTypeInfosFromTypeString("map<int,string>").get(0), 
+      new byte[]{(byte)1, (byte)2}, 0, nullSequence, false, (byte)0);
+        
+    LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+        
+    // Intialize a row result
+    HbaseMapWritable<byte[], Cell> cells = new HbaseMapWritable<byte[], Cell>();
+    cells.put("cfa:col1".getBytes(), new Cell("cfacol1".getBytes(), 0));
+    cells.put("cfa:col2".getBytes(), new Cell("cfacol2".getBytes(), 0));
+    cells.put("cfb:2".getBytes(),    new Cell("def".getBytes(), 0));
+    cells.put("cfb:-1".getBytes(),   new Cell("".getBytes(), 0));
+    cells.put("cfb:0".getBytes(),    new Cell("0".getBytes(), 0));
+    cells.put("cfb:8".getBytes(),    new Cell("abc".getBytes(), 0));
+    cells.put("cfc:col3".getBytes(), new Cell("cfccol3".getBytes(), 0));
+        
+    RowResult rr = new RowResult("test-row".getBytes(), cells);
+        
+    b.init(rr, "cfb:");
+        
+    assertEquals(
+      new Text("def"),
+      ((LazyString)b.getMapValueElement(
+        new IntWritable(2))).getWritableObject());
+    assertNull(b.getMapValueElement(new IntWritable(-1)));
+    assertEquals(
+      new Text("0"),
+      ((LazyString)b.getMapValueElement(
+        new IntWritable(0))).getWritableObject());
+    assertEquals(
+      new Text("abc"),
+      ((LazyString)b.getMapValueElement(
+        new IntWritable(8))).getWritableObject());
+    assertNull(b.getMapValueElement(new IntWritable(12345)));
+        
+    assertEquals("{0:'0',2:'def',8:'abc'}".replace('\'', '\"'),
+      SerDeUtils.getJSONString(b, oi));
+  }
+      
+  /**
+   * Test the LazyMap class with String-to-String.
+   */
+  public void testLazyHBaseCellMap2() {
+    // Map of String to String
+    Text nullSequence = new Text("\\N");
+    ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+      TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(0), 
+      new byte[]{(byte)'#', (byte)'\t'}, 0, nullSequence, false, (byte)0);
+        
+    LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+        
+    // Intialize a row result
+    HbaseMapWritable<byte[], Cell> cells =
+      new HbaseMapWritable<byte[], Cell>();
+    cells.put("cfa:col1".getBytes(), new Cell("cfacol1".getBytes(), 0));
+    cells.put("cfa:col2".getBytes(), new Cell("cfacol2".getBytes(), 0));
+    cells.put("cfb:2".getBytes(),    new Cell("d\tf".getBytes(), 0));
+    cells.put("cfb:-1".getBytes(),   new Cell("".getBytes(), 0));
+    cells.put("cfb:0".getBytes(),    new Cell("0".getBytes(), 0));
+    cells.put("cfb:8".getBytes(),    new Cell("abc".getBytes(), 0));
+    cells.put("cfc:col3".getBytes(), new Cell("cfccol3".getBytes(), 0));
+        
+    RowResult rr = new RowResult("test-row".getBytes(), cells);
+        
+    b.init(rr, "cfb:");
+        
+    assertEquals(
+      new Text("d\tf"),
+      ((LazyString)b.getMapValueElement(
+        new Text("2"))).getWritableObject());
+    assertNull(b.getMapValueElement(new Text("-1")));
+    assertEquals(
+      new Text("0"),
+      ((LazyString)b.getMapValueElement(
+        new Text("0"))).getWritableObject());
+    assertEquals(
+      new Text("abc"),
+      ((LazyString)b.getMapValueElement(
+        new Text("8"))).getWritableObject());
+    assertNull(b.getMapValueElement(new Text("-")));
+        
+    assertEquals(
+      "{'0':'0','2':'d\\tf','8':'abc'}".replace('\'', '\"'),
+      SerDeUtils.getJSONString(b, oi));
+  }
+  
+  /**
+   * Test the LazyHBaseRow class with one-for-one mappings between
+   * Hive fields and HBase columns.
+   */
+  public void testLazyHBaseRow1() {
+    List<TypeInfo> fieldTypeInfos = 
+      TypeInfoUtils.getTypeInfosFromTypeString(
+        "string,int,array<string>,map<string,string>,string");
+    List<String> fieldNames = Arrays.asList(
+      new String[]{"key", "a", "b", "c", "d"});
+    Text nullSequence = new Text("\\N");
+        
+    List<String> hbaseColumnNames = 
+      Arrays.asList(new String[]{"cfa:a", "cfa:b", "cfb:c", "cfb:d"});
+        
+    ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames,
+      fieldTypeInfos, new byte[] {' ', ':', '='},
+      nullSequence, false, false, (byte)0);
+    LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi);
+
+    HbaseMapWritable<byte[], Cell> cells =
+      new HbaseMapWritable<byte[], Cell>();
+        
+    cells.put("cfa:a".getBytes(), new Cell("123".getBytes(), 0));
+    cells.put("cfa:b".getBytes(), new Cell("a:b:c".getBytes(), 0));
+    cells.put("cfb:c".getBytes(), new Cell("d=e:f=g".getBytes(), 0));
+    cells.put("cfb:d".getBytes(), new Cell("hi".getBytes(), 0));
+    RowResult rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      ("{'key':'test-row','a':123,'b':['a','b','c'],"
+        + "'c':{'d':'e','f':'g'},'d':'hi'}").replace("'", "\""), 
+      SerDeUtils.getJSONString(o, oi));
+
+    cells.clear();
+    cells.put("cfa:a".getBytes(), new Cell("123".getBytes(), 0));
+    cells.put("cfb:c".getBytes(), new Cell("d=e:f=g".getBytes(), 0));
+    rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      ("{'key':'test-row','a':123,'b':null,"
+        + "'c':{'d':'e','f':'g'},'d':null}").replace("'", "\""),
+      SerDeUtils.getJSONString(o, oi));
+
+    cells.clear();
+    cells.put("cfa:b".getBytes(), new Cell("a".getBytes(), 0));
+    cells.put("cfb:c".getBytes(), new Cell("d=\\N:f=g:h".getBytes(), 0));
+    cells.put("cfb:d".getBytes(), new Cell("no".getBytes(), 0));
+    rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      ("{'key':'test-row','a':null,'b':['a'],"
+        + "'c':{'d':null,'f':'g','h':null},'d':'no'}").replace("'", "\""),
+      SerDeUtils.getJSONString(o, oi));
+
+    cells.clear();
+    cells.put("cfa:b".getBytes(), new Cell(":a::".getBytes(), 0));
+    cells.put("cfb:d".getBytes(), new Cell("no".getBytes(), 0));
+    rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      ("{'key':'test-row','a':null,'b':['','a','',''],"
+        + "'c':null,'d':'no'}").replace("'", "\""),
+      SerDeUtils.getJSONString(o, oi));
+
+    cells.clear();
+    cells.put("cfa:a".getBytes(), new Cell("123".getBytes(), 0));
+    cells.put("cfa:b".getBytes(), new Cell("".getBytes(), 0));
+    cells.put("cfb:c".getBytes(), new Cell("".getBytes(), 0));
+    cells.put("cfb:d".getBytes(), new Cell("".getBytes(), 0));
+    rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      "{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
+      SerDeUtils.getJSONString(o, oi));
+  }
+      
+  /**
+   * Test the LazyHBaseRow class with a mapping from a Hive field to
+   * an HBase column family.
+   */
+  public void testLazyHBaseRow2() {
+    // column family is mapped to Map<string,string>
+    List<TypeInfo> fieldTypeInfos = 
+      TypeInfoUtils.getTypeInfosFromTypeString(
+        "string,int,array<string>,map<string,string>,string");
+    List<String> fieldNames = Arrays.asList(
+      new String[]{"key", "a", "b", "c", "d"});
+    Text nullSequence = new Text("\\N");
+        
+    List<String> hbaseColumnNames = 
+      Arrays.asList(new String[]{"cfa:a", "cfa:b", "cfb:", "cfc:d"});
+        
+    ObjectInspector oi = LazyFactory.createLazyStructInspector(
+      fieldNames,
+      fieldTypeInfos,
+      new byte[] {' ', ':', '='},
+      nullSequence, false, false, (byte) 0);
+    LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi);
+
+    HbaseMapWritable<byte[], Cell> cells =
+      new HbaseMapWritable<byte[], Cell>();
+        
+    cells.put("cfa:a".getBytes(), new Cell("123".getBytes(), 0));
+    cells.put("cfa:b".getBytes(), new Cell("a:b:c".getBytes(), 0));
+    cells.put("cfb:d".getBytes(), new Cell("e".getBytes(), 0));
+    cells.put("cfb:f".getBytes(), new Cell("g".getBytes(), 0));
+    cells.put("cfc:d".getBytes(), new Cell("hi".getBytes(), 0));
+    RowResult rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      ("{'key':'test-row','a':123,'b':['a','b','c'],"
+        + "'c':{'d':'e','f':'g'},'d':'hi'}").replace("'", "\""), 
+      SerDeUtils.getJSONString(o, oi));
+
+    cells.clear();
+    cells.put("cfa:a".getBytes(), new Cell("123".getBytes(), 0));
+    cells.put("cfb:d".getBytes(), new Cell("e".getBytes(), 0));
+    cells.put("cfb:f".getBytes(), new Cell("g".getBytes(), 0));
+    rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      ("{'key':'test-row','a':123,'b':null,"
+        + "'c':{'d':'e','f':'g'},'d':null}").replace("'", "\""),
+      SerDeUtils.getJSONString(o, oi));
+
+    cells.clear();
+    cells.put("cfa:b".getBytes(), new Cell("a".getBytes(), 0));
+    cells.put("cfb:f".getBytes(), new Cell("g".getBytes(), 0));
+    cells.put("cfc:d".getBytes(), new Cell("no".getBytes(), 0));
+    rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      ("{'key':'test-row','a':null,'b':['a'],"
+        + "'c':{'f':'g'},'d':'no'}").replace("'", "\""),
+      SerDeUtils.getJSONString(o, oi));
+
+    cells.clear();
+    cells.put("cfa:b".getBytes(), new Cell(":a::".getBytes(), 0));
+    cells.put("cfc:d".getBytes(), new Cell("no".getBytes(), 0));
+    rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      ("{'key':'test-row','a':null,'b':['','a','',''],"
+        + "'c':{},'d':'no'}").replace("'", "\""),
+      SerDeUtils.getJSONString(o, oi));
+
+    cells.clear();
+    cells.put("cfa:a".getBytes(), new Cell("123".getBytes(), 0));
+    cells.put("cfa:b".getBytes(), new Cell("".getBytes(), 0));
+    cells.put("cfc:d".getBytes(), new Cell("".getBytes(), 0));
+    rr = new RowResult("test-row".getBytes(), cells);
+    o.init(rr, hbaseColumnNames);
+    assertEquals(
+      "{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
+      SerDeUtils.getJSONString(o, oi));
+  }
+}

Added: hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q (added)
+++ hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q Fri Mar 12 19:53:57 2010
@@ -0,0 +1,140 @@
+DROP TABLE hbase_table_1;
+CREATE TABLE hbase_table_1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string",
+"hbase.table.name" = "hbase_table_0"
+);
+
+DESCRIBE EXTENDED hbase_table_1;
+
+select * from hbase_table_1;
+
+EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT *;
+FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT *;
+
+DROP TABLE hbase_table_2;
+CREATE EXTERNAL TABLE hbase_table_2(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string",
+"hbase.table.name" = "hbase_table_0"
+);
+
+EXPLAIN 
+SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20;
+
+SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20;
+
+EXPLAIN 
+SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key, value;
+
+SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key,value;
+
+DROP TABLE empty_hbase_table;
+CREATE TABLE empty_hbase_table(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string"
+);
+
+DROP TABLE empty_normal_table;
+CREATE TABLE empty_normal_table(key int, value string);
+
+select * from (select count(1) as c from empty_normal_table union all select count(1) as c from empty_hbase_table) x order by c;
+select * from (select count(1) c from empty_normal_table union all select count(1) as c from hbase_table_1) x order by c;
+select * from (select count(1) c from src union all select count(1) as c from empty_hbase_table) x order by c;
+select * from (select count(1) c from src union all select count(1) as c from hbase_table_1) x order by c;
+
+CREATE TABLE hbase_table_3(key int, value string, count int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:val,cf2:count"
+);
+
+EXPLAIN 
+INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key);
+
+INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key);
+
+select count(1) from hbase_table_3;
+select * from hbase_table_3 order by key, value limit 5;
+select key, count from hbase_table_3 order by key, count desc limit 5;
+
+DROP TABLE hbase_table_4;
+CREATE TABLE hbase_table_4(key int, value1 string, value2 int, value3 int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:b,a:c,d:e"
+);
+
+INSERT OVERWRITE TABLE hbase_table_4 SELECT key, value, key+1, key+2 
+FROM src WHERE key=98 OR key=100;
+
+SELECT * FROM hbase_table_4 ORDER BY key;
+
+DROP TABLE hbase_table_5;
+CREATE EXTERNAL TABLE hbase_table_5(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:",
+"hbase.table.name" = "hbase_table_4"
+);
+
+SELECT * FROM hbase_table_5 ORDER BY key;
+
+DROP TABLE hbase_table_6;
+CREATE TABLE hbase_table_6(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:"
+);
+INSERT OVERWRITE TABLE hbase_table_6 SELECT key, map(value, key) FROM src
+WHERE key=98 OR key=100;
+
+SELECT * FROM hbase_table_6 ORDER BY key;
+
+DROP TABLE hbase_table_1;
+DROP TABLE hbase_table_2;
+DROP TABLE hbase_table_3;
+DROP TABLE hbase_table_4;
+DROP TABLE hbase_table_5;
+DROP TABLE hbase_table_6;
+DROP TABLE empty_hbase_table;
+DROP TABLE empty_normal_table;

Added: hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out (added)
+++ hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out Fri Mar 12 19:53:57 2010
@@ -0,0 +1,889 @@
+PREHOOK: query: DROP TABLE hbase_table_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string",
+"hbase.table.name" = "hbase_table_0"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE hbase_table_1(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string",
+"hbase.table.name" = "hbase_table_0"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hbase_table_1
+PREHOOK: query: DESCRIBE EXTENDED hbase_table_1
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE EXTENDED hbase_table_1
+POSTHOOK: type: DESCTABLE
+key	int	from deserializer
+value	string	from deserializer
+	 	 
+Detailed Table Information	Table(tableName:hbase_table_1, dbName:default, owner:jsichi, createTime:1267756519, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/jsichi/open/hive-trunk/build/hbase-handler/test/data/warehouse/hbase_table_1, inputFormat:org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat, outputFormat:org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.hbase.HBaseSerDe, parameters:{hbase.table.name=hbase_table_0,serialization.format=1,hbase.columns.mapping=cf:string}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{transient_lastDdlTime=1267756519,storage_handler=org.apache.hadoop.hive.hbase.HBaseStorageHandler}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE)	
+PREHOOK: query: select * from hbase_table_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_1
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-19_322_2377787749146102087/10000
+POSTHOOK: query: select * from hbase_table_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_1
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-19_322_2377787749146102087/10000
+PREHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT *
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT *
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB hbase_table_1)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Map Reduce
+      Alias -> Map Operator Tree:
+        src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Select Operator
+                expressions:
+                      expr: UDFToInteger(_col0)
+                      type: int
+                      expr: _col1
+                      type: string
+                outputColumnNames: _col0, _col1
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 1
+                  table:
+                      input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat
+                      output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat
+                      serde: org.apache.hadoop.hive.hbase.HBaseSerDe
+                      name: hbase_table_1
+
+
+PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT *
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_1
+POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT *
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_1
+PREHOOK: query: DROP TABLE hbase_table_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE hbase_table_2(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string",
+"hbase.table.name" = "hbase_table_0"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE hbase_table_2(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string",
+"hbase.table.name" = "hbase_table_0"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hbase_table_2
+PREHOOK: query: EXPLAIN 
+SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN 
+SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF hbase_table_1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF hbase_table_1))))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src))))) Y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF Y))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))) (TOK_LIMIT 20)))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        x:hbase_table_1 
+          TableScan
+            alias: hbase_table_1
+            Select Operator
+              expressions:
+                    expr: key
+                    type: int
+              outputColumnNames: _col0
+              Reduce Output Operator
+                key expressions:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                sort order: +
+                Map-reduce partition columns:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                tag: 0
+        y:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Reduce Output Operator
+                key expressions:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                sort order: +
+                Map-reduce partition columns:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                tag: 1
+                value expressions:
+                      expr: _col0
+                      type: string
+                      expr: _col1
+                      type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 {VALUE._col0} {VALUE._col1}
+          handleSkewJoin: false
+          outputColumnNames: _col2, _col3
+          Select Operator
+            expressions:
+                  expr: _col2
+                  type: string
+                  expr: _col3
+                  type: string
+            outputColumnNames: _col0, _col1
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-28_474_6112992225153930391/10002 
+            Reduce Output Operator
+              key expressions:
+                    expr: _col0
+                    type: string
+                    expr: _col1
+                    type: string
+              sort order: ++
+              tag: -1
+              value expressions:
+                    expr: _col0
+                    type: string
+                    expr: _col1
+                    type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 20
+
+
+PREHOOK: query: SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_1
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-29_174_8975920582945482917/10000
+POSTHOOK: query: SELECT Y.* 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.* FROM src) Y
+ON (x.key = Y.key)
+ORDER BY key, value LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_1
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-29_174_8975920582945482917/10000
+0	val_0
+0	val_0
+0	val_0
+10	val_10
+100	val_100
+100	val_100
+103	val_103
+103	val_103
+104	val_104
+104	val_104
+105	val_105
+11	val_11
+111	val_111
+113	val_113
+113	val_113
+114	val_114
+116	val_116
+118	val_118
+118	val_118
+119	val_119
+PREHOOK: query: EXPLAIN 
+SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key, value
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN 
+SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key, value
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF hbase_table_1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF hbase_table_1))) (TOK_WHERE (> (. (TOK_TABLE_OR_COL hbase_table_1) key) 100)))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF hbase_table_2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF hbase_table_2))) (TOK_WHERE (< (. (TOK_TABLE_OR_COL hbase_table_2) key) 120)))) Y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF Y))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        x:hbase_table_1 
+          TableScan
+            alias: hbase_table_1
+            Filter Operator
+              predicate:
+                  expr: (key > 100)
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: (key > 100)
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                  outputColumnNames: _col0
+                  Reduce Output Operator
+                    key expressions:
+                          expr: _col0
+                          type: int
+                    sort order: +
+                    Map-reduce partition columns:
+                          expr: _col0
+                          type: int
+                    tag: 0
+        y:hbase_table_2 
+          TableScan
+            alias: hbase_table_2
+            Filter Operator
+              predicate:
+                  expr: (key < 120)
+                  type: boolean
+              Filter Operator
+                predicate:
+                    expr: (key < 120)
+                    type: boolean
+                Select Operator
+                  expressions:
+                        expr: key
+                        type: int
+                        expr: value
+                        type: string
+                  outputColumnNames: _col0, _col1
+                  Reduce Output Operator
+                    key expressions:
+                          expr: _col0
+                          type: int
+                    sort order: +
+                    Map-reduce partition columns:
+                          expr: _col0
+                          type: int
+                    tag: 1
+                    value expressions:
+                          expr: _col0
+                          type: int
+                          expr: _col1
+                          type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 {VALUE._col0} {VALUE._col1}
+          handleSkewJoin: false
+          outputColumnNames: _col2, _col3
+          Select Operator
+            expressions:
+                  expr: _col2
+                  type: int
+                  expr: _col3
+                  type: string
+            outputColumnNames: _col0, _col1
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-39_652_5426194195818800442/10002 
+            Reduce Output Operator
+              key expressions:
+                    expr: _col0
+                    type: int
+                    expr: _col1
+                    type: string
+              sort order: ++
+              tag: -1
+              value expressions:
+                    expr: _col0
+                    type: int
+                    expr: _col1
+                    type: string
+      Reduce Operator Tree:
+        Extract
+          File Output Operator
+            compressed: false
+            GlobalTableId: 0
+            table:
+                input format: org.apache.hadoop.mapred.TextInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+PREHOOK: query: SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key,value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_2
+PREHOOK: Input: default@hbase_table_1
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-40_196_5786874737964073449/10000
+POSTHOOK: query: SELECT Y.*
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
+JOIN 
+(SELECT hbase_table_2.* FROM hbase_table_2 WHERE hbase_table_2.key < 120) Y
+ON (x.key = Y.key)
+ORDER BY key,value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_2
+POSTHOOK: Input: default@hbase_table_1
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-40_196_5786874737964073449/10000
+103	val_103
+104	val_104
+105	val_105
+111	val_111
+113	val_113
+114	val_114
+116	val_116
+118	val_118
+119	val_119
+PREHOOK: query: DROP TABLE empty_hbase_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE empty_hbase_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE empty_hbase_table(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE empty_hbase_table(key int, value string) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:string"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@empty_hbase_table
+PREHOOK: query: DROP TABLE empty_normal_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE empty_normal_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE empty_normal_table(key int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE empty_normal_table(key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@empty_normal_table
+PREHOOK: query: select * from (select count(1) as c from empty_normal_table union all select count(1) as c from empty_hbase_table) x order by c
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty_hbase_table
+PREHOOK: Input: default@empty_normal_table
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-53_774_4932416505718919705/10000
+POSTHOOK: query: select * from (select count(1) as c from empty_normal_table union all select count(1) as c from empty_hbase_table) x order by c
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty_hbase_table
+POSTHOOK: Input: default@empty_normal_table
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-35-53_774_4932416505718919705/10000
+0
+0
+PREHOOK: query: select * from (select count(1) c from empty_normal_table union all select count(1) as c from hbase_table_1) x order by c
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty_normal_table
+PREHOOK: Input: default@hbase_table_1
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-36-11_247_4483857956143479296/10000
+POSTHOOK: query: select * from (select count(1) c from empty_normal_table union all select count(1) as c from hbase_table_1) x order by c
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty_normal_table
+POSTHOOK: Input: default@hbase_table_1
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-36-11_247_4483857956143479296/10000
+0
+309
+PREHOOK: query: select * from (select count(1) c from src union all select count(1) as c from empty_hbase_table) x order by c
+PREHOOK: type: QUERY
+PREHOOK: Input: default@empty_hbase_table
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-36-31_650_7389757943419672684/10000
+POSTHOOK: query: select * from (select count(1) c from src union all select count(1) as c from empty_hbase_table) x order by c
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@empty_hbase_table
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-36-31_650_7389757943419672684/10000
+0
+500
+PREHOOK: query: select * from (select count(1) c from src union all select count(1) as c from hbase_table_1) x order by c
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@hbase_table_1
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-36-52_081_4218645310908641305/10000
+POSTHOOK: query: select * from (select count(1) c from src union all select count(1) as c from hbase_table_1) x order by c
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@hbase_table_1
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-36-52_081_4218645310908641305/10000
+309
+500
+PREHOOK: query: CREATE TABLE hbase_table_3(key int, value string, count int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:val,cf2:count"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE hbase_table_3(key int, value string, count int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:val,cf2:count"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hbase_table_3
+PREHOOK: query: EXPLAIN 
+INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN 
+INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key)
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF hbase_table_1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF hbase_table_1))))) x) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key)) (TOK_SELEXPR (TOK_FUNCTION count (. (TOK_TABLE_OR_COL src) key)) count)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL src) key)))) Y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL Y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB hbase_table_3)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL Y) count)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        y:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+              outputColumnNames: key
+              Group By Operator
+                aggregations:
+                      expr: count(key)
+                bucketGroup: false
+                keys:
+                      expr: key
+                      type: string
+                mode: hash
+                outputColumnNames: _col0, _col1
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col0
+                        type: string
+                  sort order: +
+                  Map-reduce partition columns:
+                        expr: _col0
+                        type: string
+                  tag: -1
+                  value expressions:
+                        expr: _col1
+                        type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE._col0)
+          bucketGroup: false
+          keys:
+                expr: KEY._col0
+                type: string
+          mode: mergepartial
+          outputColumnNames: _col0, _col1
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: string
+                  expr: _col1
+                  type: bigint
+            outputColumnNames: _col0, _col1
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-0
+    Map Reduce
+      Alias -> Map Operator Tree:
+        $INTNAME 
+            Reduce Output Operator
+              key expressions:
+                    expr: UDFToDouble(_col0)
+                    type: double
+              sort order: +
+              Map-reduce partition columns:
+                    expr: UDFToDouble(_col0)
+                    type: double
+              tag: 1
+              value expressions:
+                    expr: _col1
+                    type: bigint
+        x:hbase_table_1 
+          TableScan
+            alias: hbase_table_1
+            Select Operator
+              expressions:
+                    expr: key
+                    type: int
+                    expr: value
+                    type: string
+              outputColumnNames: _col0, _col1
+              Reduce Output Operator
+                key expressions:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                sort order: +
+                Map-reduce partition columns:
+                      expr: UDFToDouble(_col0)
+                      type: double
+                tag: 0
+                value expressions:
+                      expr: _col0
+                      type: int
+                      expr: _col1
+                      type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 {VALUE._col0} {VALUE._col1}
+            1 {VALUE._col1}
+          handleSkewJoin: false
+          outputColumnNames: _col0, _col1, _col3
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: int
+                  expr: _col1
+                  type: string
+                  expr: _col3
+                  type: bigint
+            outputColumnNames: _col0, _col1, _col2
+            Select Operator
+              expressions:
+                    expr: _col0
+                    type: int
+                    expr: _col1
+                    type: string
+                    expr: UDFToInteger(_col2)
+                    type: int
+              outputColumnNames: _col0, _col1, _col2
+              File Output Operator
+                compressed: false
+                GlobalTableId: 1
+                table:
+                    input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat
+                    output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat
+                    serde: org.apache.hadoop.hive.hbase.HBaseSerDe
+                    name: hbase_table_3
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_1
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_3
+POSTHOOK: query: INSERT OVERWRITE TABLE hbase_table_3
+SELECT x.key, x.value, Y.count 
+FROM 
+(SELECT hbase_table_1.* FROM hbase_table_1) x
+JOIN 
+(SELECT src.key, count(src.key) as count FROM src GROUP BY src.key) Y
+ON (x.key = Y.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_1
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_3
+PREHOOK: query: select count(1) from hbase_table_3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_3
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-37-27_778_7430808467238905099/10000
+POSTHOOK: query: select count(1) from hbase_table_3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_3
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-37-27_778_7430808467238905099/10000
+309
+PREHOOK: query: select * from hbase_table_3 order by key, value limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_3
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-37-35_700_409189601738780567/10000
+POSTHOOK: query: select * from hbase_table_3 order by key, value limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_3
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-37-35_700_409189601738780567/10000
+0	val_0	3
+2	val_2	1
+4	val_4	1
+5	val_5	3
+8	val_8	1
+PREHOOK: query: select key, count from hbase_table_3 order by key, count desc limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_3
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-37-41_668_5441634624522241634/10000
+POSTHOOK: query: select key, count from hbase_table_3 order by key, count desc limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_3
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-37-41_668_5441634624522241634/10000
+0	3
+2	1
+4	1
+5	3
+8	1
+PREHOOK: query: DROP TABLE hbase_table_4
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_4
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_4(key int, value1 string, value2 int, value3 int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:b,a:c,d:e"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE hbase_table_4(key int, value1 string, value2 int, value3 int) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:b,a:c,d:e"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hbase_table_4
+PREHOOK: query: INSERT OVERWRITE TABLE hbase_table_4 SELECT key, value, key+1, key+2 
+FROM src WHERE key=98 OR key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_4
+POSTHOOK: query: INSERT OVERWRITE TABLE hbase_table_4 SELECT key, value, key+1, key+2 
+FROM src WHERE key=98 OR key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_4
+PREHOOK: query: SELECT * FROM hbase_table_4 ORDER BY key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_4
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-37-57_178_5676812981426512234/10000
+POSTHOOK: query: SELECT * FROM hbase_table_4 ORDER BY key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_4
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-37-57_178_5676812981426512234/10000
+98	val_98	99	100
+100	val_100	101	102
+PREHOOK: query: DROP TABLE hbase_table_5
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_5
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE hbase_table_5(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:",
+"hbase.table.name" = "hbase_table_4"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE hbase_table_5(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "a:",
+"hbase.table.name" = "hbase_table_4"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hbase_table_5
+PREHOOK: query: SELECT * FROM hbase_table_5 ORDER BY key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_5
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-38-04_569_1715131384516433665/10000
+POSTHOOK: query: SELECT * FROM hbase_table_5 ORDER BY key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_5
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-38-04_569_1715131384516433665/10000
+98	{"b":"val_98","c":"99"}
+100	{"b":"val_100","c":"101"}
+PREHOOK: query: DROP TABLE hbase_table_6
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_6
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_6(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE hbase_table_6(key int, value map<string,string>) 
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hbase_table_6
+PREHOOK: query: INSERT OVERWRITE TABLE hbase_table_6 SELECT key, map(value, key) FROM src
+WHERE key=98 OR key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_6
+POSTHOOK: query: INSERT OVERWRITE TABLE hbase_table_6 SELECT key, map(value, key) FROM src
+WHERE key=98 OR key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_6
+PREHOOK: query: SELECT * FROM hbase_table_6 ORDER BY key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_6
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-38-18_633_3994440627035138481/10000
+POSTHOOK: query: SELECT * FROM hbase_table_6 ORDER BY key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_6
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-04_18-38-18_633_3994440627035138481/10000
+98	{"val_98":"98"}
+100	{"val_100":"100"}
+PREHOOK: query: DROP TABLE hbase_table_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@hbase_table_1
+PREHOOK: query: DROP TABLE hbase_table_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@hbase_table_2
+PREHOOK: query: DROP TABLE hbase_table_3
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@hbase_table_3
+PREHOOK: query: DROP TABLE hbase_table_4
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_4
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@hbase_table_4
+PREHOOK: query: DROP TABLE hbase_table_5
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_5
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@hbase_table_5
+PREHOOK: query: DROP TABLE hbase_table_6
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_6
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@hbase_table_6
+PREHOOK: query: DROP TABLE empty_hbase_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE empty_hbase_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@empty_hbase_table
+PREHOOK: query: DROP TABLE empty_normal_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE empty_normal_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@empty_normal_table

Added: hadoop/hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm (added)
+++ hadoop/hive/trunk/hbase-handler/src/test/templates/TestHBaseCliDriver.vm Fri Mar 12 19:53:57 2010
@@ -0,0 +1,123 @@
+package org.apache.hadoop.hive.cli;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+import java.io.*;
+import java.util.*;
+
+import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.history.HiveHistoryViewer;
+import org.apache.hadoop.hive.ql.history.HiveHistory.QueryInfo;
+import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
+import org.apache.hadoop.hive.ql.history.HiveHistory.TaskInfo;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+
+
+import org.antlr.runtime.*;
+import org.antlr.runtime.tree.*;
+
+public class $className extends TestCase {
+
+  private HBaseQTestUtil qt;
+
+  public $className(String name) {
+    super(name);
+    qt = null;
+  }
+
+  @Override
+  protected void setUp() {
+    try {
+      boolean miniMR = false;
+      if ("$clusterMode".equals("miniMR")) {
+        miniMR = true;
+      }
+
+      qt = new HBaseQTestUtil(
+        "$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()", miniMR);
+
+#foreach ($qf in $qfiles)
+      qt.addFile("$qf.getCanonicalPath()");
+#end
+    } catch (Exception e) {
+      System.out.println("Exception: " + e.getMessage());
+      e.printStackTrace();
+      System.out.flush();
+      fail("Unexpected exception in setup");
+    }
+  }
+
+  @Override
+  protected void tearDown() {
+    try {
+      qt.shutdown();
+    }
+    catch (Exception e) {
+      System.out.println("Exception: " + e.getMessage());
+      e.printStackTrace();
+      System.out.flush();
+      fail("Unexpected exception in tearDown");
+    }
+  }
+
+  public static Test suite() {
+    TestSuite suite = new TestSuite();
+#foreach ($qf in $qfiles)
+  #set ($fname = $qf.getName())
+  #set ($eidx = $fname.length() - 2)
+  #set ($tname = $fname.substring(0, $eidx))
+    suite.addTest(new $className("testCliDriver_$tname"));
+#end
+    return suite;
+  }
+
+#foreach ($qf in $qfiles)
+  #set ($fname = $qf.getName())
+  #set ($eidx = $fname.length() - 2)
+  #set ($tname = $fname.substring(0, $eidx))
+  public void testCliDriver_$tname() throws Exception {
+    try {
+      System.out.println("Begin query: " + "$fname");
+      qt.cliInit("$fname");
+      int ecode = qt.executeClient("$fname");
+      if (ecode != 0) {
+        fail("Client Execution failed with error code = " + ecode);
+      }
+      if (SessionState.get() != null) {
+        HiveHistoryViewer hv = new HiveHistoryViewer(SessionState.get()
+          .getHiveHistory().getHistFileName());
+        Map<String, QueryInfo> jobInfoMap = hv.getJobInfoMap();
+        Map<String, TaskInfo> taskInfoMap = hv.getTaskInfoMap();
+
+        if (jobInfoMap.size() != 0) {
+          String cmd = (String)jobInfoMap.keySet().toArray()[0];
+          QueryInfo ji = jobInfoMap.get(cmd);
+
+          if (!ji.hm.get(Keys.QUERY_RET_CODE.name()).equals("0")) {
+              fail("Wrong return code in hive history");
+          }
+        }
+      }
+
+      ecode = qt.checkCliDriverResults("$fname");
+      if (ecode != 0) {
+        fail("Client execution results failed with error code = " + ecode);
+      }
+    } catch (Throwable e) {
+      System.out.println("Exception: " + e.getMessage());
+      e.printStackTrace();
+      System.out.flush();
+      fail("Unexpected exception");
+    }
+
+    System.out.println("Done query: " + "$fname");
+    assertTrue("Test passed", true);
+  }
+
+#end
+}
+

Modified: hadoop/hive/trunk/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/if/hive_metastore.thrift?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/if/hive_metastore.thrift (original)
+++ hadoop/hive/trunk/metastore/if/hive_metastore.thrift Fri Mar 12 19:53:57 2010
@@ -238,5 +238,5 @@ const string META_TABLE_SERDE     = "ser
 const string META_TABLE_PARTITION_COLUMNS = "partition_columns",
 const string FILE_INPUT_FORMAT    = "file.inputformat",
 const string FILE_OUTPUT_FORMAT   = "file.outputformat",
-
+const string META_TABLE_STORAGE   = "storage_handler",
 

Modified: hadoop/hive/trunk/metastore/src/gen-cpp/hive_metastore_constants.cpp
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/gen-cpp/hive_metastore_constants.cpp?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/gen-cpp/hive_metastore_constants.cpp (original)
+++ hadoop/hive/trunk/metastore/src/gen-cpp/hive_metastore_constants.cpp Fri Mar 12 19:53:57 2010
@@ -36,6 +36,8 @@ hive_metastoreConstants::hive_metastoreC
 
   FILE_OUTPUT_FORMAT = "file.outputformat";
 
+  META_TABLE_STORAGE = "storage_handler";
+
 }
 
 }}} // namespace

Modified: hadoop/hive/trunk/metastore/src/gen-cpp/hive_metastore_constants.h
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/gen-cpp/hive_metastore_constants.h?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/gen-cpp/hive_metastore_constants.h (original)
+++ hadoop/hive/trunk/metastore/src/gen-cpp/hive_metastore_constants.h Fri Mar 12 19:53:57 2010
@@ -27,6 +27,7 @@ class hive_metastoreConstants {
   std::string META_TABLE_PARTITION_COLUMNS;
   std::string FILE_INPUT_FORMAT;
   std::string FILE_OUTPUT_FORMAT;
+  std::string META_TABLE_STORAGE;
 };
 
 extern const hive_metastoreConstants g_hive_metastore_constants;

Modified: hadoop/hive/trunk/metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java (original)
+++ hadoop/hive/trunk/metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java Fri Mar 12 19:53:57 2010
@@ -42,4 +42,6 @@ public class Constants {
 
   public static final String FILE_OUTPUT_FORMAT = "file.outputformat";
 
+  public static final String META_TABLE_STORAGE = "storage_handler";
+
 }

Modified: hadoop/hive/trunk/metastore/src/gen-php/hive_metastore_constants.php
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/gen-php/hive_metastore_constants.php?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/gen-php/hive_metastore_constants.php (original)
+++ hadoop/hive/trunk/metastore/src/gen-php/hive_metastore_constants.php Fri Mar 12 19:53:57 2010
@@ -34,4 +34,6 @@ $GLOBALS['hive_metastore_CONSTANTS']['FI
 
 $GLOBALS['hive_metastore_CONSTANTS']['FILE_OUTPUT_FORMAT'] = "file.outputformat";
 
+$GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_STORAGE'] = "storage_handler";
+
 ?>

Modified: hadoop/hive/trunk/metastore/src/gen-py/hive_metastore/constants.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/gen-py/hive_metastore/constants.py?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/gen-py/hive_metastore/constants.py (original)
+++ hadoop/hive/trunk/metastore/src/gen-py/hive_metastore/constants.py Fri Mar 12 19:53:57 2010
@@ -33,3 +33,5 @@ FILE_INPUT_FORMAT = "file.inputformat"
 
 FILE_OUTPUT_FORMAT = "file.outputformat"
 
+META_TABLE_STORAGE = "storage_handler"
+

Added: hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java (added)
+++ hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java Fri Mar 12 19:53:57 2010
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+/**
+ * HiveMetaHook defines notification methods which are invoked as part
+ * of transactions against the metastore, allowing external catalogs
+ * such as HBase to be kept in sync with Hive's metastore.
+ *
+ *<p>
+ *
+ * Implementations can use {@link MetaStoreUtils#isExternalTable} to
+ * distinguish external tables from managed tables.
+ */
+public interface HiveMetaHook {
+  /**
+   * Called before a new table definition is added to the metastore
+   * during CREATE TABLE.
+   *
+   * @param table new table definition
+   */
+  public void preCreateTable(Table table)
+    throws MetaException;
+
+  /**
+   * Called after failure adding a new table definition to the metastore
+   * during CREATE TABLE.
+   *
+   * @param table new table definition
+   */
+  public void rollbackCreateTable(Table table)
+    throws MetaException;
+
+  /**
+   * Called after successfully adding a new table definition to the metastore
+   * during CREATE TABLE.
+   *
+   * @param table new table definition
+   */
+  public void commitCreateTable(Table table)
+    throws MetaException;
+
+  /**
+   * Called before a table definition is removed from the metastore
+   * during DROP TABLE.
+   *
+   * @param table table definition
+   */
+  public void preDropTable(Table table)
+    throws MetaException;
+
+  /**
+   * Called after failure removing a table definition from the metastore
+   * during DROP TABLE.
+   *
+   * @param table table definition
+   */
+  public void rollbackDropTable(Table table)
+    throws MetaException;
+
+  /**
+   * Called after successfully removing a table definition from the metastore
+   * during DROP TABLE.
+   *
+   * @param table table definition
+   *
+   * @param deleteData whether to delete data as well; this should typically
+   * be ignored in the case of an external table
+   */
+  public void commitDropTable(Table table, boolean deleteData)
+    throws MetaException;
+}

Added: hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java?rev=922404&view=auto
==============================================================================
--- hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java (added)
+++ hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java Fri Mar 12 19:53:57 2010
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+/**
+ * HiveMetaHookLoader is responsible for loading a {@link HiveMetaHook}
+ * for a given table.
+ */
+public interface HiveMetaHookLoader {
+  /**
+   * Loads a hook for the specified table.
+   *
+   * @param tbl table of interest
+   *
+   * @return hook, or null if none registered
+   */
+  public HiveMetaHook getHook(Table tbl) throws MetaException;
+}
+
+// End HiveMetaHookLoader.java

Modified: hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Fri Mar 12 19:53:57 2010
@@ -319,9 +319,9 @@ public class HiveMetaStore extends Thrif
             tblPath = wh.getDefaultTablePath(
               tbl.getDbName(), tbl.getTableName());
           } else {
-            if (!isExternal(tbl)) {
+            if (!isExternal(tbl) && !MetaStoreUtils.isNonNativeTable(tbl)) {
               LOG.warn("Location: " + tbl.getSd().getLocation()
-                + "specified for non-external table:" + tbl.getTableName());
+                + " specified for non-external table:" + tbl.getTableName());
             }
             tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation()));
           }
@@ -418,15 +418,7 @@ public class HiveMetaStore extends Thrif
      * @return True if the table is external, otherwise false.
      */
     private boolean isExternal(Table table) {
-      if (table == null) {
-        return false;
-      }
-      Map<String, String> params = table.getParameters();
-      if (params == null) {
-        return false;
-      }
-
-      return "TRUE".equalsIgnoreCase(params.get("EXTERNAL"));
+      return MetaStoreUtils.isExternalTable(table);
     }
 
     public Table get_table(String dbname, String name) throws MetaException,

Modified: hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Fri Mar 12 19:53:57 2010
@@ -56,13 +56,17 @@ public class HiveMetaStoreClient impleme
   private boolean open = false;
   private URI metastoreUris[];
   private final boolean standAloneClient = false;
+  private HiveMetaHookLoader hookLoader;
 
   // for thrift connects
   private int retries = 5;
 
   static final private Log LOG = LogFactory.getLog("hive.metastore");
 
-  public HiveMetaStoreClient(HiveConf conf) throws MetaException {
+  public HiveMetaStoreClient(HiveConf conf, HiveMetaHookLoader hookLoader)
+    throws MetaException {
+
+    this.hookLoader = hookLoader;
     if (conf == null) {
       conf = new HiveConf(HiveMetaStoreClient.class);
     }
@@ -262,7 +266,22 @@ public class HiveMetaStoreClient impleme
    */
   public void createTable(Table tbl) throws AlreadyExistsException,
       InvalidObjectException, MetaException, NoSuchObjectException, TException {
-    client.create_table(tbl);
+    HiveMetaHook hook = getHook(tbl);
+    if (hook != null) {
+      hook.preCreateTable(tbl);
+    }
+    boolean success = false;
+    try {
+      client.create_table(tbl);
+      if (hook != null) {
+        hook.commitCreateTable(tbl);
+      }
+      success = true;
+    } finally {
+      if (!success && (hook != null)) {
+        hook.rollbackCreateTable(tbl);
+      }
+    }
   }
 
   /**
@@ -356,12 +375,34 @@ public class HiveMetaStoreClient impleme
   public void dropTable(String dbname, String name, boolean deleteData,
       boolean ignoreUknownTab) throws MetaException, TException,
       NoSuchObjectException {
+
+    Table tbl;
+    try {
+      tbl = getTable(dbname, name);
+    } catch (NoSuchObjectException e) {
+      if (!ignoreUknownTab) {
+        throw e;
+      }
+      return;
+    }
+    HiveMetaHook hook = getHook(tbl);
+    if (hook != null) {
+      hook.preDropTable(tbl);
+    }
+    boolean success = false;
     try {
       client.drop_table(dbname, name, deleteData);
+      if (hook != null) {
+        hook.commitDropTable(tbl, deleteData);
+      }
     } catch (NoSuchObjectException e) {
       if (!ignoreUknownTab) {
         throw e;
       }
+    } finally {
+      if (!success && (hook != null)) {
+        hook.rollbackDropTable(tbl);
+      }
     }
   }
 
@@ -558,4 +599,11 @@ public class HiveMetaStoreClient impleme
       throws NoSuchObjectException, MetaException, TException {
     return client.drop_partition_by_name(dbName, tableName, partName, deleteData);
   }
+
+  private HiveMetaHook getHook(Table tbl) throws MetaException {
+    if (hookLoader == null) {
+      return null;
+    }
+    return hookLoader.getHook(tbl);
+  }
 }

Modified: hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hadoop/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Fri Mar 12 19:53:57 2010
@@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.Constants;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 
 public class MetaStoreUtils {
@@ -336,13 +338,13 @@ public class MetaStoreUtils {
             schema
                 .getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_LOCATION));
     t.getSd().setInputFormat(
-        schema.getProperty(
-            org.apache.hadoop.hive.metastore.api.Constants.FILE_INPUT_FORMAT,
-            org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName()));
+      schema.getProperty(
+        org.apache.hadoop.hive.metastore.api.Constants.FILE_INPUT_FORMAT,
+        org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName()));
     t.getSd().setOutputFormat(
-        schema.getProperty(
-            org.apache.hadoop.hive.metastore.api.Constants.FILE_OUTPUT_FORMAT,
-            org.apache.hadoop.mapred.SequenceFileOutputFormat.class.getName()));
+      schema.getProperty(
+        org.apache.hadoop.hive.metastore.api.Constants.FILE_OUTPUT_FORMAT,
+        org.apache.hadoop.mapred.SequenceFileOutputFormat.class.getName()));
     t.setPartitionKeys(new ArrayList<FieldSchema>());
     t.setDbName(MetaStoreUtils.DEFAULT_DATABASE_NAME);
     String part_cols_str = schema
@@ -601,19 +603,19 @@ public class MetaStoreUtils {
     String inputFormat = sd.getInputFormat();
     if (inputFormat == null || inputFormat.length() == 0) {
       inputFormat = org.apache.hadoop.mapred.SequenceFileInputFormat.class
-          .getName();
+        .getName();
     }
     schema.setProperty(
-        org.apache.hadoop.hive.metastore.api.Constants.FILE_INPUT_FORMAT,
-        inputFormat);
+      org.apache.hadoop.hive.metastore.api.Constants.FILE_INPUT_FORMAT,
+      inputFormat);
     String outputFormat = sd.getOutputFormat();
     if (outputFormat == null || outputFormat.length() == 0) {
       outputFormat = org.apache.hadoop.mapred.SequenceFileOutputFormat.class
-          .getName();
+        .getName();
     }
     schema.setProperty(
-        org.apache.hadoop.hive.metastore.api.Constants.FILE_OUTPUT_FORMAT,
-        outputFormat);
+      org.apache.hadoop.hive.metastore.api.Constants.FILE_OUTPUT_FORMAT,
+      outputFormat);
     schema.setProperty(
         org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME,
         tableName);
@@ -814,4 +816,29 @@ public class MetaStoreUtils {
         "generated by TypeInfoUtils.getFieldSchemaFromTypeInfo");
   }
 
+  /**
+   * Determines whether a table is an external table.
+   *
+   * @param table table of interest
+   *
+   * @return true if external
+   */
+  public static boolean isExternalTable(Table table) {
+    if (table == null) {
+      return false;
+    }
+    Map<String, String> params = table.getParameters();
+    if (params == null) {
+      return false;
+    }
+
+    return "TRUE".equalsIgnoreCase(params.get("EXTERNAL"));
+  }
+
+  public static boolean isNonNativeTable(Table table) {
+    if (table == null) {
+      return false;
+    }
+    return (table.getParameters().get(Constants.META_TABLE_STORAGE) != null);
+  }
 }

Modified: hadoop/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (original)
+++ hadoop/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java Fri Mar 12 19:53:57 2010
@@ -58,7 +58,7 @@ public class TestHiveMetaStore extends T
     hiveConf.set("hive.key4", "0");
 
     try {
-      client = new HiveMetaStoreClient(hiveConf);
+      client = new HiveMetaStoreClient(hiveConf, null);
     } catch (Throwable e) {
       System.err.println("Unable to open the metastore");
       System.err.println(StringUtils.stringifyException(e));

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Fri Mar 12 19:53:57 2010
@@ -61,6 +61,7 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreChecker;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
 import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
@@ -216,9 +217,7 @@ public class DDLTask extends Task<DDLWor
     Table tbl = db.getTable(addPartitionDesc.getDbName(), addPartitionDesc
         .getTableName());
 
-    if (tbl.isView()) {
-      throw new HiveException("Cannot use ALTER TABLE on a view");
-    }
+    validateAlterTableType(tbl);
 
     // If the add partition was created with IF NOT EXISTS, then we should
     // not throw an error if the specified part does exist.
@@ -242,6 +241,16 @@ public class DDLTask extends Task<DDLWor
     return 0;
   }
 
+  private void validateAlterTableType(Table tbl) throws HiveException {
+    if (tbl.isView()) {
+      throw new HiveException("Cannot use ALTER TABLE on a view");
+    }
+
+    if (tbl.isNonNative()) {
+      throw new HiveException("Cannot use ALTER TABLE on a non-native table");
+    }
+  }
+  
   /**
    * MetastoreCheck, see if the data in the metastore matches what is on the
    * dfs. Current version checks for tables and partitions that are either
@@ -960,9 +969,8 @@ public class DDLTask extends Task<DDLWor
     Table tbl = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, alterTbl
         .getOldName());
 
-    if (tbl.isView()) {
-      throw new HiveException("Cannot use ALTER TABLE on a view");
-    }
+    validateAlterTableType(tbl);
+
     Table oldTbl = tbl.copy();
 
     if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) {
@@ -1278,16 +1286,15 @@ public class DDLTask extends Task<DDLWor
       tbl.setNumBuckets(crtTbl.getNumBuckets());
     }
 
+    if (crtTbl.getStorageHandler() != null) {
+      tbl.setProperty(
+        org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_STORAGE,
+        crtTbl.getStorageHandler());
+    }
+    HiveStorageHandler storageHandler = tbl.getStorageHandler();
+
     if (crtTbl.getSerName() != null) {
       tbl.setSerializationLib(crtTbl.getSerName());
-      if (crtTbl.getMapProp() != null) {
-        Iterator<Entry<String, String>> iter = crtTbl.getMapProp().entrySet()
-            .iterator();
-        while (iter.hasNext()) {
-          Entry<String, String> m = (Entry<String, String>) iter.next();
-          tbl.setSerdeParam(m.getKey(), m.getValue());
-        }
-      }
     } else {
       if (crtTbl.getFieldDelim() != null) {
         tbl.setSerdeParam(Constants.FIELD_DELIM, crtTbl.getFieldDelim());
@@ -1311,15 +1318,31 @@ public class DDLTask extends Task<DDLWor
       }
     }
 
-    /**
+    if (crtTbl.getMapProp() != null) {
+      Iterator<Entry<String, String>> iter = crtTbl.getMapProp().entrySet()
+        .iterator();
+      while (iter.hasNext()) {
+        Entry<String, String> m = (Entry<String, String>) iter.next();
+        tbl.setSerdeParam(m.getKey(), m.getValue());
+      }
+    }
+
+    /*
      * We use LazySimpleSerDe by default.
      * 
      * If the user didn't specify a SerDe, and any of the columns are not simple
      * types, we will have to use DynamicSerDe instead.
      */
     if (crtTbl.getSerName() == null) {
-      LOG.info("Default to LazySimpleSerDe for table " + crtTbl.getTableName());
-      tbl.setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
+      if (storageHandler == null) {
+        LOG.info("Default to LazySimpleSerDe for table " + crtTbl.getTableName());
+        tbl.setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
+      } else {
+        String serDeClassName = storageHandler.getSerDeClass().getName();
+        LOG.info("Use StorageHandler-supplied " + serDeClassName
+          + " for table " + crtTbl.getTableName());
+        tbl.setSerializationLib(serDeClassName);
+      }
     } else {
       // let's validate that the serde exists
       validateSerDe(crtTbl.getSerName());
@@ -1344,6 +1367,11 @@ public class DDLTask extends Task<DDLWor
     tbl.setInputFormatClass(crtTbl.getInputFormat());
     tbl.setOutputFormatClass(crtTbl.getOutputFormat());
 
+    tbl.getTTable().getSd().setInputFormat(
+      tbl.getInputFormatClass().getName());
+    tbl.getTTable().getSd().setOutputFormat(
+      tbl.getOutputFormatClass().getName());
+
     if (crtTbl.isExternal()) {
       tbl.setProperty("EXTERNAL", "TRUE");
       tbl.setTableType(TableType.EXTERNAL_TABLE);

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Fri Mar 12 19:53:57 2010
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.io.Hive
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.MapredWork;
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -1130,12 +1131,18 @@ public class ExecDriver extends Task<Map
     // The input file does not exist, replace it by a empty file
     Class<? extends HiveOutputFormat> outFileFormat = null;
 
+    TableDesc tableDesc;
     if (isEmptyPath) {
-      outFileFormat = work.getPathToPartitionInfo().get(path).getTableDesc()
-          .getOutputFileFormatClass();
+      tableDesc = work.getPathToPartitionInfo().get(path).getTableDesc();
     } else {
-      outFileFormat = work.getAliasToPartnInfo().get(alias).getTableDesc()
-          .getOutputFileFormatClass();
+      tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc();
+    }
+    outFileFormat = tableDesc.getOutputFileFormatClass();
+
+    if (tableDesc.isNonNative()) {
+      FileInputFormat.addInputPaths(job, path);
+      LOG.info("Add a non-native table " + path);
+      return numEmptyPaths;
     }
 
     // create a dummy empty file in a new directory

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=922404&r1=922403&r2=922404&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Fri Mar 12 19:53:57 2010
@@ -64,16 +64,21 @@ public class FetchOperator implements Se
 
   public FetchOperator() {
   }
-  
+
   public FetchOperator(FetchWork work, JobConf job) {
     this.work = work;
     initialize(job);
   }
-  
+
   public void initialize(JobConf job) {
     this.job = job;
     tblDataDone = false;
     rowWithPart = new Object[2];
+    if (work.getTblDesc() != null) {
+      isNativeTable = !work.getTblDesc().isNonNative();
+    } else {
+      isNativeTable = true;
+    }
   }
 
   public FetchWork getWork() {
@@ -116,12 +121,13 @@ public class FetchOperator implements Se
     this.tblDataDone = tblDataDone;
   }
 
+  private boolean isNativeTable;
   private FetchWork work;
   private int splitNum;
   private PartitionDesc currPart;
   private TableDesc currTbl;
   private boolean tblDataDone;
-  
+
   private transient RecordReader<WritableComparable, Writable> currRecReader;
   private transient InputSplit[] inputSplits;
   private transient InputFormat inputFormat;
@@ -145,7 +151,7 @@ public class FetchOperator implements Se
       Class inputFormatClass, Configuration conf) throws IOException {
     if (!inputFormats.containsKey(inputFormatClass)) {
       try {
-        InputFormat<WritableComparable, Writable> newInstance = 
+        InputFormat<WritableComparable, Writable> newInstance =
           (InputFormat<WritableComparable, Writable>) ReflectionUtils
             .newInstance(inputFormatClass, conf);
         inputFormats.put(inputFormatClass, newInstance);
@@ -193,15 +199,19 @@ public class FetchOperator implements Se
         if (!tblDataDone) {
           currPath = work.getTblDirPath();
           currTbl = work.getTblDesc();
-          FileSystem fs = currPath.getFileSystem(job);
-          if (fs.exists(currPath)) {
-            FileStatus[] fStats = fs.listStatus(currPath);
-            for (FileStatus fStat : fStats) {
-              if (fStat.getLen() > 0) {
-                tblDataDone = true;
-                break;
+          if (isNativeTable) {
+            FileSystem fs = currPath.getFileSystem(job);
+            if (fs.exists(currPath)) {
+              FileStatus[] fStats = fs.listStatus(currPath);
+              for (FileStatus fStat : fStats) {
+                if (fStat.getLen() > 0) {
+                  tblDataDone = true;
+                  break;
+                }
               }
             }
+          } else {
+            tblDataDone = true;
           }
 
           if (!tblDataDone) {
@@ -261,6 +271,7 @@ public class FetchOperator implements Se
         tmp = currPart.getTableDesc();
       }
       inputFormat = getInputFormatFromCache(tmp.getInputFileFormatClass(), job);
+      Utilities.copyTableJobPropertiesToConf(tmp, job);
       inputSplits = inputFormat.getSplits(job, 1);
       splitNum = 0;
       serde = tmp.getDeserializerClass().newInstance();
@@ -291,7 +302,7 @@ public class FetchOperator implements Se
 
   /**
    * Get the next row. The fetch context is modified appropriately.
-   * 
+   *
    **/
   public InspectableObject getNextRow() throws IOException {
     try {
@@ -321,10 +332,10 @@ public class FetchOperator implements Se
       throw new IOException(e);
     }
   }
-  
+
   /**
    * Clear the context, if anything needs to be done.
-   * 
+   *
    **/
   public void clearFetchContext() throws HiveException {
     try {
@@ -340,10 +351,10 @@ public class FetchOperator implements Se
           + org.apache.hadoop.util.StringUtils.stringifyException(e));
     }
   }
-  
+
   /**
-   * used for bucket map join. there is a hack for getting partitionDesc. 
-   * bucket map join right now only allow one partition present in bucket map join. 
+   * used for bucket map join. there is a hack for getting partitionDesc.
+   * bucket map join right now only allow one partition present in bucket map join.
    */
   public void setupContext (Iterator<Path> iterPath, Iterator<PartitionDesc> iterPartDesc) {
     this.iterPath = iterPath;



Mime
View raw message