hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r926818 - in /hadoop/hive/trunk: ./ contrib/src/test/results/clientnegative/ hbase-handler/src/test/queries/ hbase-handler/src/test/results/ ql/src/java/org/apache/hadoop/hive/ql/io/
Date Tue, 23 Mar 2010 22:29:44 GMT
Author: namit
Date: Tue Mar 23 22:29:44 2010
New Revision: 926818

URL: http://svn.apache.org/viewvc?rev=926818&view=rev
Log:
HIVE-1257. join between hbase and other tables does not work
(John Sichi via namit)


Added:
    hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q
    hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=926818&r1=926817&r2=926818&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Mar 23 22:29:44 2010
@@ -291,6 +291,9 @@ Trunk -  Unreleased
     HIVE-1268. Cannot start metastore thrift server on a specific port
     (bc Wong via namit)
 
+    HIVE-1257. join between hbase and other tables does not work
+    (John Sichi via namit)
+
 Release 0.5.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out?rev=926818&r1=926817&r2=926818&view=diff
==============================================================================
--- hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out (original)
+++ hadoop/hive/trunk/contrib/src/test/results/clientnegative/serde_regex.q.out Tue Mar 23
22:29:44 2010
@@ -51,12 +51,12 @@ STAGE PLANS:
           columns: host string, identity string, user string, time string, request string,
status int, size int, referer string, agent string
           if not exists: false
           input format: org.apache.hadoop.mapred.TextInputFormat
-          serde properties:
-            input.regex ([^ ]*) ([^ ]*) ([^ ]*) (-|\[[^\]]*\]) ([^ "]*|"[^"]*") (-|[0-9]*)
(-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))?
-            output.format.string %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s
           # buckets: -1
           output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
           serde name: org.apache.hadoop.hive.contrib.serde2.RegexSerDe
+          serde properties:
+            input.regex ([^ ]*) ([^ ]*) ([^ ]*) (-|\[[^\]]*\]) ([^ "]*|"[^"]*") (-|[0-9]*)
(-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))?
+            output.format.string %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s
           name: serde_regex
           isExternal: false
 

Added: hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q?rev=926818&view=auto
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q (added)
+++ hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_joins.q Tue Mar 23 22:29:44 2010
@@ -0,0 +1,64 @@
+DROP TABLE users;
+DROP TABLE states;
+DROP TABLE countries;
+
+-- From HIVE-1257
+
+CREATE TABLE users(key string, state string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:state,info:country,info:country_id"
+);
+
+CREATE TABLE states(key string, name string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "state:name"
+);
+
+CREATE TABLE countries(key string, name string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:name,info:country,info:country_id"
+);
+
+INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
+FROM src WHERE key=100;
+
+INSERT OVERWRITE TABLE states SELECT 'IA', 'Iowa'
+FROM src WHERE key=100;
+
+INSERT OVERWRITE TABLE countries SELECT 'USA', 'United States', 'USA', 1
+FROM src WHERE key=100;
+
+set hive.input.format = org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id);
+
+SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key);
+
+set hive.input.format = org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country);
+
+SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id);
+
+SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key);
+
+DROP TABLE users;
+DROP TABLE states;
+DROP TABLE countries;

Added: hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out?rev=926818&view=auto
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out (added)
+++ hadoop/hive/trunk/hbase-handler/src/test/results/hbase_joins.q.out Tue Mar 23 22:29:44
2010
@@ -0,0 +1,202 @@
+PREHOOK: query: DROP TABLE users
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE users
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE states
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE states
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE countries
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE countries
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: -- From HIVE-1257
+
+CREATE TABLE users(key string, state string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:state,info:country,info:country_id"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- From HIVE-1257
+
+CREATE TABLE users(key string, state string, country string, country_id int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:state,info:country,info:country_id"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@users
+PREHOOK: query: CREATE TABLE states(key string, name string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "state:name"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE states(key string, name string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "state:name"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@states
+PREHOOK: query: CREATE TABLE countries(key string, name string, country string, country_id
int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:name,info:country,info:country_id"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE countries(key string, name string, country string, country_id
int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "info:name,info:country,info:country_id"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@countries
+PREHOOK: query: INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
+FROM src WHERE key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@users
+POSTHOOK: query: INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA', 0
+FROM src WHERE key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@users
+PREHOOK: query: INSERT OVERWRITE TABLE states SELECT 'IA', 'Iowa'
+FROM src WHERE key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@states
+POSTHOOK: query: INSERT OVERWRITE TABLE states SELECT 'IA', 'Iowa'
+FROM src WHERE key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@states
+PREHOOK: query: INSERT OVERWRITE TABLE countries SELECT 'USA', 'United States', 'USA', 1
+FROM src WHERE key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@countries
+POSTHOOK: query: INSERT OVERWRITE TABLE countries SELECT 'USA', 'United States', 'USA', 1
+FROM src WHERE key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@countries
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-19_117_4712777556867957470/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-19_117_4712777556867957470/10000
+user1	USA	United States	USA
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-25_489_4149172260159935049/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-25_489_4149172260159935049/10000
+user1	USA	United States	USA
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-32_285_599526889736525802/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-32_285_599526889736525802/10000
+PREHOOK: query: SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@states
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-38_500_2952626450057612534/10000
+POSTHOOK: query: SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@states
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-38_500_2952626450057612534/10000
+user1	IA	Iowa
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-45_911_3744508358740860232/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country = c.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-45_911_3744508358740860232/10000
+user1	USA	United States	USA
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-52_375_497455375799515705/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c
+ON (u.country = c.country)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-52_375_497455375799515705/10000
+user1	USA	United States	USA
+PREHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@countries
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-59_778_6209720937134856434/10000
+POSTHOOK: query: SELECT u.key, u.country, c.name, c.key FROM users u JOIN countries c 
+ON (u.country_id = c.country_id)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@countries
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-31-59_778_6209720937134856434/10000
+PREHOOK: query: SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@users
+PREHOOK: Input: default@states
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-32-06_717_1044214443379190261/10000
+POSTHOOK: query: SELECT u.key, u.state, s.name FROM users u JOIN states s 
+ON (u.state = s.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@users
+POSTHOOK: Input: default@states
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-23_12-32-06_717_1044214443379190261/10000
+user1	IA	Iowa
+PREHOOK: query: DROP TABLE users
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE users
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@users
+PREHOOK: query: DROP TABLE states
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE states
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@states
+PREHOOK: query: DROP TABLE countries
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE countries
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@countries

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=926818&r1=926817&r2=926818&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
(original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
Tue Mar 23 22:29:44 2010
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.shims.HadoopShims.CombineFileInputFormatShim;
 import org.apache.hadoop.hive.shims.HadoopShims.InputSplitShim;
@@ -260,6 +261,11 @@ public class CombineHiveInputFormat<K ex
       Class inputFormatClass = part.getInputFileFormatClass();
       InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);
 
+      TableDesc tableDesc = part.getTableDesc();
+      if ((tableDesc != null) && tableDesc.isNonNative()) {
+        return super.getSplits(job, numSplits);
+      }
+
       if ((inputFormat instanceof TextInputFormat) &&
           ((new CompressionCodecFactory(job)).getCodec(tstPath) != null)) {
         return super.getSplits(job, numSplits);

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=926818&r1=926817&r2=926818&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Tue Mar
23 22:29:44 2010
@@ -220,12 +220,10 @@ public class HiveInputFormat<K extends W
     InputFormat inputFormat = getInputFormatFromCache(inputFormatClass,
         cloneJobConf);
 
-    Path [] paths = FileInputFormat.getInputPaths(job);
-    // for now we only get one path for splits which access a non-native
-    // table; should probably add a corresponding assertion
-    PartitionDesc part = getPartitionDescFromPath(
-      pathToPartitionInfo, paths[0]);
-    Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf);
+    PartitionDesc part = pathToPartitionInfo.get(hsplit.getPath().toString());
+    if ((part != null) && (part.getTableDesc() != null)) {
+      Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf);
+    }
     return new HiveRecordReader(inputFormat.getRecordReader(inputSplit,
         cloneJobConf, reporter));
   }



Mime
View raw message