hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1557893 - in /hive/trunk: itests/qtest/pom.xml ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java ql/src/test/queries/clientpositive/root_dir_external_table.q ql/src/test/results/clientpositive/root_dir_external_table.q.out
Date Tue, 14 Jan 2014 00:43:52 GMT
Author: hashutosh
Date: Tue Jan 14 00:43:52 2014
New Revision: 1557893

URL: http://svn.apache.org/r1557893
Log:
HIVE-5032 : Enable hive creating external table at the root directory of DFS (Shuaishuai Nie
via Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/root_dir_external_table.q
    hive/trunk/ql/src/test/results/clientpositive/root_dir_external_table.q.out
Modified:
    hive/trunk/itests/qtest/pom.xml
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java

Modified: hive/trunk/itests/qtest/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest/pom.xml?rev=1557893&r1=1557892&r2=1557893&view=diff
==============================================================================
--- hive/trunk/itests/qtest/pom.xml (original)
+++ hive/trunk/itests/qtest/pom.xml Tue Jan 14 00:43:52 2014
@@ -36,7 +36,7 @@
     <run_disabled>false</run_disabled>
     <clustermode></clustermode>
     <execute.beeline.tests>false</execute.beeline.tests>
-    <minimr.query.files>stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q</minimr.query.files>
+    <minimr.query.files>stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q</minimr.query.files>
     <minimr.query.negative.files>cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q</minimr.query.negative.files>
     <minitez.query.files>tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q</minitez.query.files>
     <minitez.query.files.shared>join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q</minitez.query.files.shared>

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=1557893&r1=1557892&r2=1557893&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Tue Jan 14
00:43:52 2014
@@ -380,23 +380,22 @@ public final class HiveFileFormatUtils {
     }
 
     if (part == null) {
-      String dirStr = dir.toString();
-      int dirPathIndex = dirPath.lastIndexOf(Path.SEPARATOR);
-      int dirStrIndex = dirStr.lastIndexOf(Path.SEPARATOR);
-      while (dirPathIndex >= 0 && dirStrIndex >= 0) {
-        dirStr = dirStr.substring(0, dirStrIndex);
-        dirPath = dirPath.substring(0, dirPathIndex);
-        //first try full match
-        part = pathToPartitionInfo.get(dirStr);
+      Path curPath = new Path(dir.toUri().getPath()).getParent();
+      dir = dir.getParent();
+      while (dir != null) {
+
+        // first try full match
+        part = pathToPartitionInfo.get(dir.toString());
         if (part == null) {
-          // LOG.warn("exact match not found, try ripping input path's theme and authority");
-          part = pathToPartitionInfo.get(dirPath);
+
+          // exact match not found, try ripping input path's scheme and authority
+          part = pathToPartitionInfo.get(curPath.toString());
         }
         if (part != null) {
           break;
         }
-        dirPathIndex = dirPath.lastIndexOf(Path.SEPARATOR);
-        dirStrIndex = dirStr.lastIndexOf(Path.SEPARATOR);
+        dir = dir.getParent();
+        curPath = curPath.getParent();
       }
     }
     return part;

Added: hive/trunk/ql/src/test/queries/clientpositive/root_dir_external_table.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/root_dir_external_table.q?rev=1557893&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/root_dir_external_table.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/root_dir_external_table.q Tue Jan 14 00:43:52
2014
@@ -0,0 +1,11 @@
+dfs ${system:test.dfs.mkdir} hdfs:///tmp/test;
+
+insert overwrite directory "hdfs:///tmp/test" select key from src where (key < 20) order
by key;
+
+dfs -cp /tmp/test/000000_0 /000000_0;
+dfs -rmr hdfs:///tmp/test;
+
+create external table roottable (key string) row format delimited fields terminated by '\\t'
stored as textfile location 'hdfs:///';
+select count(*) from roottable;
+
+dfs -rmr /000000_0;
\ No newline at end of file

Added: hive/trunk/ql/src/test/results/clientpositive/root_dir_external_table.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/root_dir_external_table.q.out?rev=1557893&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/root_dir_external_table.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/root_dir_external_table.q.out Tue Jan 14
00:43:52 2014
@@ -0,0 +1,21 @@
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@roottable
+PREHOOK: query: select count(*) from roottable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@roottable
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from roottable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@roottable
+#### A masked pattern was here ####
+20
+#### A masked pattern was here ####



Mime
View raw message