hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1576166 - in /hive/branches/branch-0.13/ql/src: java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java test/queries/clientpositive/exim_hidden_files.q test/results/clientpositive/exim_hidden_files.q.out
Date Tue, 11 Mar 2014 01:25:37 GMT
Author: hashutosh
Date: Tue Mar 11 01:25:37 2014
New Revision: 1576166

URL: http://svn.apache.org/r1576166
Log:
HIVE-6591 : Importing a table containing hidden dirs fails (Ashutosh Chauhan via Vikram Dixit)

Added:
    hive/branches/branch-0.13/ql/src/test/queries/clientpositive/exim_hidden_files.q
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/exim_hidden_files.q.out
Modified:
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1576166&r1=1576165&r2=1576166&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
Tue Mar 11 01:25:37 2014
@@ -23,6 +23,7 @@ import org.apache.commons.lang.StringUti
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -62,10 +63,22 @@ public class LoadSemanticAnalyzer extend
 
   public static FileStatus[] matchFilesOrDir(FileSystem fs, Path path)
       throws IOException {
-    FileStatus[] srcs = fs.globStatus(path);
+    FileStatus[] srcs = fs.globStatus(path, new PathFilter() {
+              @Override
+              public boolean accept(Path p) {
+                String name = p.getName();
+                return name.equals("_metadata") ? true : !name.startsWith("_") &&
!name.startsWith(".");
+              }
+            });
     if ((srcs != null) && srcs.length == 1) {
       if (srcs[0].isDir()) {
-        srcs = fs.listStatus(srcs[0].getPath());
+        srcs = fs.listStatus(srcs[0].getPath(), new PathFilter() {
+          @Override
+          public boolean accept(Path p) {
+            String name = p.getName();
+            return !name.startsWith("_") && !name.startsWith(".");
+          }
+        });
       }
     }
     return (srcs);

Added: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/exim_hidden_files.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/exim_hidden_files.q?rev=1576166&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/exim_hidden_files.q (added)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/exim_hidden_files.q Tue Mar
11 01:25:37 2014
@@ -0,0 +1,22 @@
+set hive.test.mode=true;
+set hive.test.mode.prefix=;
+set hive.test.mode.nosamplelist=exim_department,exim_employee;
+
+create table exim_employee ( emp_id int) partitioned by (emp_country string);
+load data local inpath "../../data/files/test.dat" into table exim_employee partition (emp_country="in");
	
+
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/exim_employee/emp_country=in/_logs;
+dfs -touchz ${system:test.warehouse.dir}/exim_employee/emp_country=in/_logs/job.xml;
+export table exim_employee to 'ql/test/data/exports/exim_employee';
+drop table exim_employee;
+
+create database importer;
+use importer;
+
+import from 'ql/test/data/exports/exim_employee';
+describe formatted exim_employee;
+select * from exim_employee;
+dfs -rmr target/tmp/ql/test/data/exports/exim_employee;
+drop table exim_employee;
+drop database importer;
+use default;

Added: hive/branches/branch-0.13/ql/src/test/results/clientpositive/exim_hidden_files.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/exim_hidden_files.q.out?rev=1576166&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/exim_hidden_files.q.out (added)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/exim_hidden_files.q.out Tue
Mar 11 01:25:37 2014
@@ -0,0 +1,116 @@
+PREHOOK: query: create table exim_employee ( emp_id int) partitioned by (emp_country string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table exim_employee ( emp_id int) partitioned by (emp_country string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@exim_employee
+PREHOOK: query: load data local inpath "../../data/files/test.dat" into table exim_employee
partition (emp_country="in")
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@exim_employee
+POSTHOOK: query: load data local inpath "../../data/files/test.dat" into table exim_employee
partition (emp_country="in")
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@exim_employee
+POSTHOOK: Output: default@exim_employee@emp_country=in
+PREHOOK: query: export table exim_employee to 'ql/test/data/exports/exim_employee'
+PREHOOK: type: EXPORT
+PREHOOK: Input: default@exim_employee@emp_country=in
+#### A masked pattern was here ####
+POSTHOOK: query: export table exim_employee to 'ql/test/data/exports/exim_employee'
+POSTHOOK: type: EXPORT
+POSTHOOK: Input: default@exim_employee@emp_country=in
+#### A masked pattern was here ####
+PREHOOK: query: drop table exim_employee
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@exim_employee
+PREHOOK: Output: default@exim_employee
+POSTHOOK: query: drop table exim_employee
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@exim_employee
+POSTHOOK: Output: default@exim_employee
+PREHOOK: query: create database importer
+PREHOOK: type: CREATEDATABASE
+POSTHOOK: query: create database importer
+POSTHOOK: type: CREATEDATABASE
+PREHOOK: query: use importer
+PREHOOK: type: SWITCHDATABASE
+POSTHOOK: query: use importer
+POSTHOOK: type: SWITCHDATABASE
+PREHOOK: query: import from 'ql/test/data/exports/exim_employee'
+PREHOOK: type: IMPORT
+POSTHOOK: query: import from 'ql/test/data/exports/exim_employee'
+POSTHOOK: type: IMPORT
+POSTHOOK: Output: importer@exim_employee
+POSTHOOK: Output: importer@exim_employee@emp_country=in
+PREHOOK: query: describe formatted exim_employee
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted exim_employee
+POSTHOOK: type: DESCTABLE
+# col_name            	data_type           	comment             
+	 	 
+emp_id              	int                 	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+emp_country         	string              	None                
+	 	 
+# Detailed Table Information	 	 
+Database:           	importer            	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+#### A masked pattern was here ####
+Table Type:         	MANAGED_TABLE       	 
+Table Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   
+PREHOOK: query: select * from exim_employee
+PREHOOK: type: QUERY
+PREHOOK: Input: importer@exim_employee
+PREHOOK: Input: importer@exim_employee@emp_country=in
+#### A masked pattern was here ####
+POSTHOOK: query: select * from exim_employee
+POSTHOOK: type: QUERY
+POSTHOOK: Input: importer@exim_employee
+POSTHOOK: Input: importer@exim_employee@emp_country=in
+#### A masked pattern was here ####
+1	in
+2	in
+3	in
+4	in
+5	in
+6	in
+#### A masked pattern was here ####
+PREHOOK: query: drop table exim_employee
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: importer@exim_employee
+PREHOOK: Output: importer@exim_employee
+POSTHOOK: query: drop table exim_employee
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: importer@exim_employee
+POSTHOOK: Output: importer@exim_employee
+PREHOOK: query: drop database importer
+PREHOOK: type: DROPDATABASE
+PREHOOK: Input: database:importer
+PREHOOK: Output: database:importer
+POSTHOOK: query: drop database importer
+POSTHOOK: type: DROPDATABASE
+POSTHOOK: Input: database:importer
+POSTHOOK: Output: database:importer
+PREHOOK: query: use default
+PREHOOK: type: SWITCHDATABASE
+POSTHOOK: query: use default
+POSTHOOK: type: SWITCHDATABASE



Mime
View raw message