hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1468445 - in /hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/test/org/apache/hadoop/hive/ql/exec/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Tue, 16 Apr 2013 14:51:58 GMT
Author: hashutosh
Date: Tue Apr 16 14:51:58 2013
New Revision: 1468445

URL: http://svn.apache.org/r1468445
Log:
HIVE-4275 : Hive does not differentiate scheme and authority in file uris (Vikram Dixit via
Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority.q
    hive/trunk/ql/src/test/results/clientpositive/schemeAuthority.q.out
Modified:
    hive/trunk/build-common.xml
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java

Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1468445&r1=1468444&r2=1468445&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Tue Apr 16 14:51:58 2013
@@ -59,7 +59,7 @@
   <property name="test.output" value="true"/>
   <property name="test.junit.output.format" value="xml"/>
   <property name="test.junit.output.usefile" value="true"/>
-  <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,operatorhook.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q"/>
+  <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,operatorhook.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q"/>
   <property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q"
/>
   <property name="test.silent" value="true"/>
   <property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1468445&r1=1468444&r2=1468445&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Tue Apr 16 14:51:58
2013
@@ -138,7 +138,6 @@ public class MapOperator extends Operato
     public void setOp(Operator<? extends OperatorDesc> op) {
       this.op = op;
     }
-
   }
 
   private static class MapOpCtx {
@@ -468,8 +467,8 @@ public class MapOperator extends Operato
 
   public void setChildren(Configuration hconf) throws HiveException {
 
-    Path fpath = new Path((new Path(HiveConf.getVar(hconf,
-        HiveConf.ConfVars.HADOOPMAPFILENAME))).toUri().getPath());
+    Path fpath = new Path(HiveConf.getVar(hconf,
+        HiveConf.ConfVars.HADOOPMAPFILENAME));
 
     ArrayList<Operator<? extends OperatorDesc>> children =
       new ArrayList<Operator<? extends OperatorDesc>>();
@@ -481,7 +480,7 @@ public class MapOperator extends Operato
     try {
       for (String onefile : conf.getPathToAliases().keySet()) {
         MapOpCtx opCtx = initObjectInspector(conf, hconf, onefile, convertedOI);
-        Path onepath = new Path(new Path(onefile).toUri().getPath());
+        Path onepath = new Path(onefile);
         List<String> aliases = conf.getPathToAliases().get(onefile);
 
         for (String onealias : aliases) {
@@ -513,7 +512,7 @@ public class MapOperator extends Operato
         // didn't find match for input file path in configuration!
         // serious problem ..
         LOG.error("Configuration does not have any alias for path: "
-            + fpath.toUri().getPath());
+            + fpath.toUri());
         throw new HiveException("Configuration and input path are inconsistent");
       }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1468445&r1=1468444&r2=1468445&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Apr 16 14:51:58
2013
@@ -690,11 +690,6 @@ public final class Utilities {
     return new PartitionDesc(part, tblDesc);
   }
 
-  public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator<?>
work) {
-    mr.addMapWork(tbl.getDataLocation().getPath(), alias, work, new PartitionDesc(
-        getTableDesc(tbl), (LinkedHashMap<String, String>) null));
-  }
-
   private static String getOpTreeSkel_helper(Operator<?> op, String indent) {
     if (op == null) {
       return "";

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1468445&r1=1468444&r2=1468445&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Tue Apr 16 14:51:58
2013
@@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
 import org.apache.hadoop.hive.ql.plan.FilterDesc;
 import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
 import org.apache.hadoop.hive.ql.plan.PlanUtils;
 import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
 import org.apache.hadoop.hive.ql.plan.ScriptDesc;
@@ -132,6 +133,11 @@ public class TestExecDriver extends Test
     mr = PlanUtils.getMapRedWork();
   }
 
+  public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator<?>
work) {
+    mr.addMapWork(tbl.getDataLocation().toString(), alias, work, new PartitionDesc(
+        Utilities.getTableDesc(tbl), null));
+  }
+
   private static void fileDiff(String datafile, String testdir) throws Exception {
     String testFileDir = conf.get("test.data.files");
     System.out.println(testFileDir);
@@ -190,7 +196,7 @@ public class TestExecDriver extends Test
     Operator<FilterDesc> op1 = OperatorFactory.get(getTestFilterDesc("key"),
         op2);
 
-    Utilities.addMapWork(mr, src, "a", op1);
+    addMapWork(mr, src, "a", op1);
   }
 
   @SuppressWarnings("unchecked")
@@ -209,7 +215,7 @@ public class TestExecDriver extends Test
     Operator<FilterDesc> op1 = OperatorFactory.get(getTestFilterDesc("key"),
         op2);
 
-    Utilities.addMapWork(mr, src, "a", op1);
+    addMapWork(mr, src, "a", op1);
   }
 
   @SuppressWarnings("unchecked")
@@ -226,7 +232,7 @@ public class TestExecDriver extends Test
         Utilities.makeList(getStringColumn("value")), outputColumns, true,
         -1, 1, -1));
 
-    Utilities.addMapWork(mr, src, "a", op1);
+    addMapWork(mr, src, "a", op1);
     mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
     mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
 
@@ -254,7 +260,7 @@ public class TestExecDriver extends Test
         .makeList(getStringColumn("key"), getStringColumn("value")),
         outputColumns, false, -1, 1, -1));
 
-    Utilities.addMapWork(mr, src, "a", op1);
+    addMapWork(mr, src, "a", op1);
     mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
     mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
 
@@ -287,7 +293,7 @@ public class TestExecDriver extends Test
         Utilities.makeList(getStringColumn("value")), outputColumns, true,
         Byte.valueOf((byte) 0), 1, -1));
 
-    Utilities.addMapWork(mr, src, "a", op1);
+    addMapWork(mr, src, "a", op1);
     mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
     mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
 
@@ -296,7 +302,7 @@ public class TestExecDriver extends Test
         Utilities.makeList(getStringColumn("key")), outputColumns, true,
         Byte.valueOf((byte) 1), Integer.MAX_VALUE, -1));
 
-    Utilities.addMapWork(mr, src2, "b", op2);
+    addMapWork(mr, src2, "b", op2);
     mr.getTagToValueDesc().add(op2.getConf().getValueSerializeInfo());
 
     // reduce side work
@@ -338,7 +344,7 @@ public class TestExecDriver extends Test
         .makeList(getStringColumn("key"), getStringColumn("value")),
         outputColumns), op0);
 
-    Utilities.addMapWork(mr, src, "a", op4);
+    addMapWork(mr, src, "a", op4);
     mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
     mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
 
@@ -375,7 +381,7 @@ public class TestExecDriver extends Test
         .makeList(getStringColumn("key"), getStringColumn("value")),
         outputColumns), op0);
 
-    Utilities.addMapWork(mr, src, "a", op4);
+    addMapWork(mr, src, "a", op4);
     mr.setKeyDesc(op0.getConf().getKeySerializeInfo());
     mr.getTagToValueDesc().add(op0.getConf().getValueSerializeInfo());
 
@@ -414,7 +420,7 @@ public class TestExecDriver extends Test
         .makeList(getStringColumn("key"), getStringColumn("value")),
         outputColumns), op0);
 
-    Utilities.addMapWork(mr, src, "a", op4);
+    addMapWork(mr, src, "a", op4);
     mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
     mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo());
 

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java?rev=1468445&r1=1468444&r2=1468445&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java Tue Apr 16 14:51:58
2013
@@ -320,7 +320,7 @@ public class TestOperators extends TestC
       aliases.add("b");
       LinkedHashMap<String, ArrayList<String>> pathToAliases =
         new LinkedHashMap<String, ArrayList<String>>();
-      pathToAliases.put("/testDir", aliases);
+      pathToAliases.put("hdfs:///testDir", aliases);
 
       // initialize pathToTableInfo
       // Default: treat the table as a single column "col"
@@ -328,7 +328,7 @@ public class TestOperators extends TestC
       PartitionDesc pd = new PartitionDesc(td, null);
       LinkedHashMap<String, org.apache.hadoop.hive.ql.plan.PartitionDesc> pathToPartitionInfo
=
         new LinkedHashMap<String, org.apache.hadoop.hive.ql.plan.PartitionDesc>();
-      pathToPartitionInfo.put("/testDir", pd);
+      pathToPartitionInfo.put("hdfs:///testDir", pd);
 
       // initialize aliasToWork
       CollectDesc cd = new CollectDesc(Integer.valueOf(1));

Added: hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority.q?rev=1468445&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/schemeAuthority.q Tue Apr 16 14:51:58 2013
@@ -0,0 +1,15 @@
+dfs -mkdir file:///tmp/test;
+dfs -mkdir hdfs:///tmp/test;
+
+create external table dynPart (key string) partitioned by (value string) row format delimited
fields terminated by '\\t' stored as textfile;
+insert overwrite local directory "/tmp/test" select key from src where (key = 10) order by
key;
+insert overwrite directory "/tmp/test" select key from src where (key = 20) order by key;
+alter table dynPart add partition (value='0') location 'file:///tmp/test';
+alter table dynPart add partition (value='1') location 'hdfs:///tmp/test';
+select count(*) from dynPart;
+select key from dynPart;
+select key from src where (key = 10) order by key;
+select key from src where (key = 20) order by key;
+
+dfs -rmr file:///tmp/test;
+dfs -rmr hdfs:///tmp/test;

Added: hive/trunk/ql/src/test/results/clientpositive/schemeAuthority.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/schemeAuthority.q.out?rev=1468445&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/schemeAuthority.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/schemeAuthority.q.out Tue Apr 16 14:51:58
2013
@@ -0,0 +1,77 @@
+PREHOOK: query: create external table dynPart (key string) partitioned by (value string)
row format delimited fields terminated by '\\t' stored as textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create external table dynPart (key string) partitioned by (value string)
row format delimited fields terminated by '\\t' stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dynPart
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@dynpart
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Output: default@dynpart@value=0
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@dynpart
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Output: default@dynpart@value=1
+PREHOOK: query: select count(*) from dynPart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dynpart
+PREHOOK: Input: default@dynpart@value=0
+PREHOOK: Input: default@dynpart@value=1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from dynPart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Input: default@dynpart@value=0
+POSTHOOK: Input: default@dynpart@value=1
+#### A masked pattern was here ####
+2
+PREHOOK: query: select key from dynPart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dynpart
+PREHOOK: Input: default@dynpart@value=0
+PREHOOK: Input: default@dynpart@value=1
+#### A masked pattern was here ####
+POSTHOOK: query: select key from dynPart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dynpart
+POSTHOOK: Input: default@dynpart@value=0
+POSTHOOK: Input: default@dynpart@value=1
+#### A masked pattern was here ####
+20
+10
+PREHOOK: query: select key from src where (key = 10) order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from src where (key = 10) order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+10
+PREHOOK: query: select key from src where (key = 20) order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from src where (key = 20) order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+20
+#### A masked pattern was here ####



Mime
View raw message