hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From athu...@apache.org
Subject svn commit: r776145 - in /hadoop/hive/trunk: ./ data/files/ ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Tue, 19 May 2009 01:03:50 GMT
Author: athusoo
Date: Tue May 19 01:03:50 2009
New Revision: 776145

URL: http://svn.apache.org/viewvc?rev=776145&view=rev
Log:
HIVE-488. Fix load partition for table with multiple partition columns.
(Prasad Chakka via athusoo)


Added:
    hadoop/hive/trunk/data/files/test.dat
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/loadpart1.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/loadpart1.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=776145&r1=776144&r2=776145&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue May 19 01:03:50 2009
@@ -135,6 +135,9 @@
     HIVE-485. Fix join not to assume all columns are strings.
     (Namit Jain via zshao)
 
+    HIVE-488. Fix load partition for table with multiple partition columns.
+    (Prasad Chakka via athusoo)
+
 Release 0.3.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Added: hadoop/hive/trunk/data/files/test.dat
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/test.dat?rev=776145&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/test.dat (added)
+++ hadoop/hive/trunk/data/files/test.dat Tue May 19 01:03:50 2009
@@ -0,0 +1,6 @@
+1
+2
+3
+4
+5
+6
\ No newline at end of file

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=776145&r1=776144&r2=776145&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue May 19
01:03:50 2009
@@ -770,7 +770,17 @@
           // point of no return
           boolean b = fs.delete(destf, true);
           LOG.debug("Deleting:"+destf.toString()+",Status:"+b);
+          // create the parent directory otherwise rename can fail if the parent doesn't
exist
+          if (!fs.mkdirs(destf.getParent())) {
+            throw new HiveException("Unable to create destination directory: " 
+                  + destf.getParent().toString());
+          }
+          
           b = fs.rename(tmppath, destf);
+          if (!b) {
+            throw new HiveException("Unable to move results to destination directory: " 
+                + destf.getParent().toString());
+          }
           LOG.debug("Renaming:"+tmppath.toString()+",Status:"+b);
 
       } catch (IOException e) {

Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=776145&r1=776144&r2=776145&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Tue May 19 01:03:50
2009
@@ -243,14 +243,14 @@
         part_spec.put("ds", ds);
         part_spec.put("hr", hr);
         // System.out.println("Loading partition with spec: " + part_spec);
-        db.createPartition(srcpart, part_spec);
+        //db.createPartition(srcpart, part_spec);
         fpath = new Path(testFiles, "kv1.txt");
         newfpath = new Path(tmppath, "kv1.txt");
         fs.copyFromLocalFile(false, true, fpath, newfpath);
         fpath = newfpath;
         //db.loadPartition(fpath, srcpart.getName(), part_spec, true);
         runLoadCmd("LOAD DATA INPATH '" +  newfpath.toString() +
-                   "' INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='" + hr +"')");
+                   "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='" + hr
+"')");
       }
     }
     ArrayList<String> bucketCols = new ArrayList<String>();

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/loadpart1.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/loadpart1.q?rev=776145&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/loadpart1.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/loadpart1.q Tue May 19 01:03:50 2009
@@ -0,0 +1,11 @@
+drop table hive_test_src;
+drop table hive_test_dst;
+create table hive_test_src ( col1 string ) stored as textfile ;
+load data local inpath '../data/files/test.dat' overwrite into table hive_test_src ;
+create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string)
stored as sequencefile;
+insert overwrite table hive_test_dst partition ( pcol1='test_part', pcol2='test_part') select
col1 from hive_test_src ;
+select * from hive_test_dst where pcol1='test_part' and pcol2='test_part';
+insert overwrite table hive_test_dst partition ( pcol1='test_part', pcol2='test_part') select
col1 from hive_test_src ;
+select * from hive_test_dst where pcol1='test_part' and pcol2='test_part';
+drop table hive_test_src;
+drop table hive_test_dst;

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/loadpart1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/loadpart1.q.out?rev=776145&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/loadpart1.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/loadpart1.q.out Tue May 19 01:03:50
2009
@@ -0,0 +1,31 @@
+query: drop table hive_test_src
+query: drop table hive_test_dst
+query: create table hive_test_src ( col1 string ) stored as textfile
+query: load data local inpath '../data/files/test.dat' overwrite into table hive_test_src
+query: create table hive_test_dst ( col1 string ) partitioned by ( pcol1 string , pcol2 string)
stored as sequencefile
+query: insert overwrite table hive_test_dst partition ( pcol1='test_part', pcol2='test_part')
select col1 from hive_test_src
+Input: default/hive_test_src
+Output: default/hive_test_dst/pcol1=test_part/pcol2=test_part
+query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part'
+Input: default/hive_test_dst/pcol1=test_part/pcol2=test_part
+Output: /data/users/pchakka/workspace/oshive/ql/../build/ql/tmp/608894653/647768977.10000
+1	test_part	test_part
+2	test_part	test_part
+3	test_part	test_part
+4	test_part	test_part
+5	test_part	test_part
+6	test_part	test_part
+query: insert overwrite table hive_test_dst partition ( pcol1='test_part', pcol2='test_part')
select col1 from hive_test_src
+Input: default/hive_test_src
+Output: default/hive_test_dst/pcol1=test_part/pcol2=test_part
+query: select * from hive_test_dst where pcol1='test_part' and pcol2='test_part'
+Input: default/hive_test_dst/pcol1=test_part/pcol2=test_part
+Output: /data/users/pchakka/workspace/oshive/ql/../build/ql/tmp/255138744/205167628.10000
+1	test_part	test_part
+2	test_part	test_part
+3	test_part	test_part
+4	test_part	test_part
+5	test_part	test_part
+6	test_part	test_part
+query: drop table hive_test_src
+query: drop table hive_test_dst



Mime
View raw message