hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From athu...@apache.org
Subject svn commit: r773705 - in /hadoop/hive/trunk: ./ cli/src/java/org/apache/hadoop/hive/cli/ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/queries/clientnegative/...
Date Mon, 11 May 2009 22:00:29 GMT
Author: athusoo
Date: Mon May 11 22:00:28 2009
New Revision: 773705

URL: http://svn.apache.org/viewvc?rev=773705&view=rev
Log:
HIVE-469. Fix errors for cases where the partition or the
table is empty. (Namit Jain via athusoo)


Added:
    hadoop/hive/trunk/ql/src/test/queries/clientnegative/input3.q
    hadoop/hive/trunk/ql/src/test/queries/clientnegative/input4.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/input23.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/input24.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/input25.q
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/input26.q
    hadoop/hive/trunk/ql/src/test/results/clientnegative/input3.q.out
    hadoop/hive/trunk/ql/src/test/results/clientnegative/input4.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input23.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input24.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input25.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input26.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/nullgroup3.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup3.q.out

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon May 11 22:00:28 2009
@@ -153,6 +153,9 @@
 
     HIVE-447. Fix tests on hudson. (Ashish Thusoo via zshao)
 
+    HIVE-469. Fix errors for cases where the partition or the
+    table is empty. (Namit Jain via athusoo)
+
 Release 0.3.0 - 2009-04-20
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (original)
+++ hadoop/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java Mon May 11 22:00:28
2009
@@ -174,6 +174,11 @@
       long start = System.currentTimeMillis();
 
       ret = qp.run(cmd);
+      if (ret != 0) {
+        qp.close();
+        return ret;
+      }
+        
       Vector<String> res = new Vector<String>();
       while (qp.getResults(res)) {
       	for (String r:res) {

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Mon May 11 22:00:28
2009
@@ -179,6 +179,10 @@
       // Do semantic analysis and plan generation
       sem.analyze(tree, ctx);
       LOG.info("Semantic Analysis Completed");
+      
+      // validate the plan
+      sem.validate();
+
       plan = new QueryPlan(command, sem);
       return (0);
     } catch (SemanticException e) {

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Mon May 11
22:00:28 2009
@@ -40,15 +40,18 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.plan.mapredWork;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.partitionDesc;
+import org.apache.hadoop.hive.ql.plan.tableDesc;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
 import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
 import org.apache.hadoop.hive.ql.io.*;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.log4j.BasicConfigurator;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.PropertyConfigurator;
 import org.apache.log4j.varia.NullAppender;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 
 public class ExecDriver extends Task<mapredWork> implements Serializable {
 
@@ -331,11 +334,6 @@
 
     Utilities.setMapRedWork(job, work);
 
-    for (String onefile : work.getPathToAliases().keySet()) {
-      LOG.info("Adding input file " + onefile);
-      FileInputFormat.addInputPaths(job, onefile);
-    }
-
     String hiveScratchDir = HiveConf.getVar(job, HiveConf.ConfVars.SCRATCHDIR);
     Path jobScratchDir = new Path(hiveScratchDir + Utilities.randGen.nextInt());
     FileOutputFormat.setOutputPath(job, jobScratchDir);
@@ -366,27 +364,7 @@
     boolean success = false;
 
     try {
-      // if the input is empty exit gracefully
-      Path[] inputPaths = FileInputFormat.getInputPaths(job);
-      boolean emptyInput = true;
-      for (Path inputP : inputPaths) {
-        FileSystem inputFs = inputP.getFileSystem(job);
-        if (!inputFs.exists(inputP))
-          continue;
-
-        FileStatus[] fStats = inputFs.listStatus(inputP);
-        for (FileStatus fStat : fStats) {
-          if (fStat.getLen() > 0) {
-            emptyInput = false;
-            break;
-          }
-        }
-      }
-
-      if (emptyInput) {
-        console.printInfo("Job need not be submitted: no output: Success");
-        return 0;
-      }
+      addInputPaths(job, work, hiveScratchDir);
 
       // remove the pwd from conf file so that job tracker doesn't show this logs
       String pwd = job.get(HiveConf.ConfVars.METASTOREPWD.varname);
@@ -643,4 +621,80 @@
     mapredWork w = getWork();
     return w.getReducer() != null;
   }
+
+  private void addInputPaths(JobConf job, mapredWork work, String hiveScratchDir) throws
Exception {
+    FileSystem inpFs = FileSystem.get(job);
+    int numEmptyPaths = 0;
+    
+    // If the query references non-existent partitions
+    if (work.getPathToAliases().isEmpty() &&
+        !work.getAliasToWork().isEmpty()) {
+      String oneAlias = (String)work.getAliasToWork().keySet().toArray()[0];
+      
+      Class<? extends HiveOutputFormat> outFileFormat = (Class<? extends HiveOutputFormat>)Class.forName("org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat");
+      
+      String newFile = "file:" + hiveScratchDir + File.separator + (++numEmptyPaths);
+      Path newPath = new Path(newFile);
+      LOG.info("Changed input file to " + newPath.toString());
+      
+      // add a dummy work
+      Map<String, ArrayList<String>> pathToAliases = work.getPathToAliases();
+      ArrayList<String> newList = new ArrayList<String>();
+      newList.add(oneAlias);
+      pathToAliases.put(newPath.toString(), newList);
+      
+      Map<String,partitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo();
+      partitionDesc pDesc = new partitionDesc();
+      
+      tableDesc tDesc = new tableDesc(LazySimpleSerDe.class,
+                                      SequenceFileInputFormat.class,
+                                      SequenceFileOutputFormat.class,
+                                      new Properties());
+      
+      pDesc.setTableDesc(tDesc);
+      pathToPartitionInfo.put(newPath.toString(), pDesc);
+      
+      RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newPath,
Text.class, false, new Properties(), null);
+      recWriter.close(false);
+      FileInputFormat.addInputPaths(job, newPath.toString());
+    }
+    else {
+      for (String onefile : work.getPathToAliases().keySet()) {
+        LOG.info("Adding input file " + onefile);
+        
+        // If the input file does not exist, replace it by a empty file
+        Path dirPath = new Path(onefile);
+        boolean emptyInput = true;
+        
+        if (inpFs.exists(dirPath)) {
+          FileStatus[] fStats = inpFs.listStatus(dirPath);
+          if (fStats.length > 0)
+            emptyInput = false;
+        }
+        
+        if (emptyInput) {
+          Class<? extends HiveOutputFormat> outFileFormat = work.getPathToPartitionInfo().get(onefile).getTableDesc().getOutputFileFormatClass();
+          
+          String newFile = "file:" + hiveScratchDir + File.separator + (++numEmptyPaths);
+          Path newPath = new Path(newFile);
+          LOG.info("Changed input file to " + newPath.toString());
+          
+          // toggle the work
+          Map<String, ArrayList<String>> pathToAliases = work.getPathToAliases();
+          pathToAliases.put(newPath.toString(), pathToAliases.get(onefile));
+          pathToAliases.remove(onefile);
+          
+          Map<String,partitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo();
+          pathToPartitionInfo.put(newPath.toString(), pathToPartitionInfo.get(onefile));
+          pathToPartitionInfo.remove(onefile);
+          
+          onefile = newPath.toString();
+          RecordWriter recWriter = outFileFormat.newInstance().getHiveRecordWriter(job, newPath,
Text.class, false, new Properties(), null);
+          recWriter.close(false);
+        }
+        
+        FileInputFormat.addInputPaths(job, onefile);
+      }      
+    }
+  }
 }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
(original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
Mon May 11 22:00:28 2009
@@ -83,6 +83,10 @@
     scratchDir = ctx.getScratchDir();
     analyzeInternal(ast, ctx);
   }
+
+  public void validate() throws SemanticException {
+    // Implementations may choose to override this
+  }
   
   public List<Task<? extends Serializable>> getRootTasks() {
     return rootTasks;

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Mon May 11
22:00:28 2009
@@ -77,6 +77,7 @@
   UNION_NOTIN_SUBQ("Top level Union is not supported currently; use a subquery for the union"),
   INVALID_INPUT_FORMAT_TYPE("Input Format must implement InputFormat"),
   INVALID_OUTPUT_FORMAT_TYPE("Output Format must implement HiveOutputFormat, otherwise it
should be either IgnoreKeyTextOutputFormat or SequenceFileOutputFormat"),
+  NO_VALID_PARTN("The query does not reference any valid partition. To run this query, set
hive.mapred.mode=nonstrict"),
   NON_BUCKETED_TABLE("Sampling Expression Needed for Non-Bucketed Table");
 
   private String mesg;

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Mon
May 11 22:00:28 2009
@@ -3816,6 +3816,31 @@
     return newParameters;
   }
   
+  public void validate() throws SemanticException {
+    // Check if the plan contains atleast one path.
+
+    // validate all tasks
+    for(Task<? extends Serializable> rootTask: rootTasks)
+      validate(rootTask);
+  }
+
+  private void validate(Task<? extends Serializable> task) throws SemanticException
{
+    if ((task instanceof MapRedTask) || (task instanceof ExecDriver)) {
+      mapredWork work = (mapredWork)task.getWork();
+
+      if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")) {
+        if ((work.getPathToAliases() == null) || (work.getPathToAliases().isEmpty()))
+          throw new SemanticException(ErrorMsg.NO_VALID_PARTN.getMsg());
+      }
+    }
+
+    if (task.getChildTasks() == null)
+      return;
+
+    for (Task<? extends Serializable> childTask :  task.getChildTasks())
+      validate(childTask);
+  }
+
   @Override
   public Set<ReadEntity> getInputs() {
     return inputs;

Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/input3.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/input3.q?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/input3.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/input3.q Mon May 11 22:00:28 2009
@@ -0,0 +1,7 @@
+set hive.mapred.mode=strict;
+
+select * from (
+  select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' limit 5
+    union all
+  select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5
+)subq;

Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/input4.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/input4.q?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/input4.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/input4.q Mon May 11 22:00:28 2009
@@ -0,0 +1,5 @@
+set hive.mapred.mode=strict;
+
+select * from srcpart a join
+  (select b.key, count(1) as count from srcpart b where b.ds = '2008-04-08' and b.hr = '14'
group by b.key) subq
+  where a.ds = '2008-04-08' and a.hr = '11' limit 10;

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input23.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input23.q?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input23.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input23.q Mon May 11 22:00:28 2009
@@ -0,0 +1,5 @@
+explain extended
+ select * from srcpart a join srcpart b where a.ds = '2008-04-08' and a.hr = '11' and b.ds
= '2008-04-08' and b.hr = '14' limit 5;
+
+select * from srcpart a join srcpart b where a.ds = '2008-04-08' and a.hr = '11' and b.ds
= '2008-04-08' and b.hr = '14' limit 5;
+

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input24.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input24.q?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input24.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input24.q Mon May 11 22:00:28 2009
@@ -0,0 +1,9 @@
+drop table tst;
+create table tst(a int, b int) partitioned by (d string);
+alter table tst add partition (d='2009-01-01');
+explain
+select count(1) from tst x where x.d='2009-01-01';
+
+select count(1) from tst x where x.d='2009-01-01';
+
+drop table tst;

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input25.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input25.q?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input25.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input25.q Mon May 11 22:00:28 2009
@@ -0,0 +1,19 @@
+drop table tst;
+create table tst(a int, b int) partitioned by (d string);
+alter table tst add partition (d='2009-01-01');
+alter table tst add partition (d='2009-02-02');
+
+explain
+select * from (
+  select * from tst x where x.d='2009-01-01' limit 10
+    union all
+  select * from tst x where x.d='2009-02-02' limit 10
+) subq;
+
+select * from (
+  select * from tst x where x.d='2009-01-01' limit 10
+    union all
+  select * from tst x where x.d='2009-02-02' limit 10
+) subq;
+
+drop table tst;

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input26.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input26.q?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input26.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input26.q Mon May 11 22:00:28 2009
@@ -0,0 +1,12 @@
+explain
+select * from (
+  select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit
5
+    union all
+  select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5
+)subq;
+
+select * from (
+  select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit
5
+    union all
+  select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5
+)subq;

Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/nullgroup3.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/nullgroup3.q?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/nullgroup3.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/nullgroup3.q Mon May 11 22:00:28
2009
@@ -28,3 +28,6 @@
 explain
 select count(1) from tstparttbl2;
 select count(1) from tstparttbl2;
+
+DROP TABLE tstparttbl;
+DROP TABLE tstparttbl2;

Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/input3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/input3.q.out?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/input3.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/input3.q.out Mon May 11 22:00:28
2009
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: The query does not reference any valid partition. To
run this query, set hive.mapred.mode=nonstrict

Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/input4.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/input4.q.out?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/input4.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/input4.q.out Mon May 11 22:00:28
2009
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: In strict mode, cartesian product is not allowed. If
you really want to perform the operation, set hive.mapred.mode=nonstrict

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/input23.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input23.q.out?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input23.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input23.q.out Mon May 11 22:00:28
2009
@@ -0,0 +1,114 @@
+query: explain extended
+ select * from srcpart a join srcpart b where a.ds = '2008-04-08' and a.hr = '11' and b.ds
= '2008-04-08' and b.hr = '14' limit 5
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF srcpart a) (TOK_TABREF srcpart b))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE
(and (and (and (= (. (TOK_TABLE_OR_COL a) ds) '2008-04-08') (= (. (TOK_TABLE_OR_COL a) hr)
'11')) (= (. (TOK_TABLE_OR_COL b) ds) '2008-04-08')) (= (. (TOK_TABLE_OR_COL b) hr) '14')))
(TOK_LIMIT 5)))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        b 
+            Reduce Output Operator
+              sort order: 
+              tag: 1
+              value expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+                    expr: ds
+                    type: string
+                    expr: hr
+                    type: string
+        a 
+            Reduce Output Operator
+              sort order: 
+              tag: 0
+              value expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+                    expr: ds
+                    type: string
+                    expr: hr
+                    type: string
+      Needs Tagging: true
+      Path -> Alias:
+        file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11

+      Path -> Partition:
+        file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11

+          Partition
+            partition values:
+              ds 2008-04-08
+              hr 11
+          
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              properties:
+                name srcpart
+                columns.types string:string
+                serialization.ddl struct srcpart { string key, string value}
+                serialization.format 1
+                columns key,value
+                partition_columns ds/hr
+                bucket_count -1
+                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                location file:/data/users/njain/hive5/hive5/build/ql/test/data/warehouse/srcpart
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: srcpart
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 {VALUE.0} {VALUE.1} {VALUE.2} {VALUE.3}
+            1 {VALUE.0} {VALUE.1} {VALUE.2} {VALUE.3}
+          Filter Operator
+            predicate:
+                expr: ((((2 = '2008-04-08') and (3 = '11')) and (6 = '2008-04-08')) and (7
= '14'))
+                type: boolean
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: string
+                    expr: 1
+                    type: string
+                    expr: 2
+                    type: string
+                    expr: 3
+                    type: string
+                    expr: 4
+                    type: string
+                    expr: 5
+                    type: string
+                    expr: 6
+                    type: string
+                    expr: 7
+                    type: string
+              Limit
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  directory: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/57557850/240891616.10001.insclause-0
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      properties:
+                        columns a.key,a.value,a.ds,a.hr,b.key,b.value,b.ds,b.hr
+                        serialization.format 1
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 5
+
+
+query: select * from srcpart a join srcpart b where a.ds = '2008-04-08' and a.hr = '11' and
b.ds = '2008-04-08' and b.hr = '14' limit 5
+Input: default/srcpart/ds=2008-04-08/hr=11
+Output: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/738588790/856672066.10000

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/input24.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input24.q.out?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input24.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input24.q.out Mon May 11 22:00:28
2009
@@ -0,0 +1,61 @@
+query: drop table tst
+query: create table tst(a int, b int) partitioned by (d string)
+query: alter table tst add partition (d='2009-01-01')
+query: explain
+select count(1) from tst x where x.d='2009-01-01'
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF tst x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (= (. (TOK_TABLE_OR_COL x) d)
'2009-01-01'))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        x 
+            Select Operator
+              expressions:
+                    expr: d
+                    type: string
+              Filter Operator
+                predicate:
+                    expr: (0 = '2009-01-01')
+                    type: boolean
+                Group By Operator
+                  aggregations:
+                        expr: count(1)
+                  mode: hash
+                  Reduce Output Operator
+                    sort order: 
+                    tag: -1
+                    value expressions:
+                          expr: 0
+                          type: bigint
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations:
+                expr: count(VALUE.0)
+          mode: mergepartial
+          Select Operator
+            expressions:
+                  expr: 0
+                  type: bigint
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+query: select count(1) from tst x where x.d='2009-01-01'
+Input: default/tst/d=2009-01-01
+Output: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/321060343/178308884.10000
+0
+query: drop table tst

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/input25.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input25.q.out?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input25.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input25.q.out Mon May 11 22:00:28
2009
@@ -0,0 +1,147 @@
+query: drop table tst
+query: create table tst(a int, b int) partitioned by (d string)
+query: alter table tst add partition (d='2009-01-01')
+query: alter table tst add partition (d='2009-02-02')
+query: explain
+select * from (
+  select * from tst x where x.d='2009-01-01' limit 10
+    union all
+  select * from tst x where x.d='2009-02-02' limit 10
+) subq
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF tst x))
(TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))
(TOK_WHERE (= (. (TOK_TABLE_OR_COL x) d) '2009-01-01')) (TOK_LIMIT 10))) (TOK_QUERY (TOK_FROM
(TOK_TABREF tst x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR
TOK_ALLCOLREF)) (TOK_WHERE (= (. (TOK_TABLE_OR_COL x) d) '2009-02-02')) (TOK_LIMIT 10))))
subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery1:subq-subquery1:x 
+            Filter Operator
+              predicate:
+                  expr: (d = '2009-01-01')
+                  type: boolean
+              Select Operator
+                expressions:
+                      expr: a
+                      type: int
+                      expr: b
+                      type: int
+                      expr: d
+                      type: string
+                Limit
+                  Reduce Output Operator
+                    sort order: 
+                    tag: -1
+                    value expressions:
+                          expr: 0
+                          type: int
+                          expr: 1
+                          type: int
+                          expr: 2
+                          type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  name: binary_table
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        /data/users/njain/hive5/hive5/build/ql/tmp/221582197/417273265.10002 
+          Union
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: int
+                    expr: 1
+                    type: int
+                    expr: 2
+                    type: string
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+        /data/users/njain/hive5/hive5/build/ql/tmp/221582197/417273265.10003 
+          Union
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: int
+                    expr: 1
+                    type: int
+                    expr: 2
+                    type: string
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-3
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery2:subq-subquery2:x 
+            Filter Operator
+              predicate:
+                  expr: (d = '2009-02-02')
+                  type: boolean
+              Select Operator
+                expressions:
+                      expr: a
+                      type: int
+                      expr: b
+                      type: int
+                      expr: d
+                      type: string
+                Limit
+                  Reduce Output Operator
+                    sort order: 
+                    tag: -1
+                    value expressions:
+                          expr: 0
+                          type: int
+                          expr: 1
+                          type: int
+                          expr: 2
+                          type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  name: binary_table
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+query: select * from (
+  select * from tst x where x.d='2009-01-01' limit 10
+    union all
+  select * from tst x where x.d='2009-02-02' limit 10
+) subq
+Input: default/tst/d=2009-01-01
+Input: default/tst/d=2009-02-02
+Output: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/128845781/511626368.10000
+query: drop table tst

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/input26.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input26.q.out?rev=773705&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input26.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input26.q.out Mon May 11 22:00:28
2009
@@ -0,0 +1,160 @@
+query: explain
+select * from (
+  select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit
5
+    union all
+  select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5
+)subq
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF srcpart
a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))
(TOK_WHERE (and (= (. (TOK_TABLE_OR_COL a) ds) '2008-04-08') (= (. (TOK_TABLE_OR_COL a) hr)
'11'))) (TOK_ORDERBY (. (TOK_TABLE_OR_COL a) key)) (TOK_LIMIT 5))) (TOK_QUERY (TOK_FROM (TOK_TABREF
srcpart b)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR
TOK_ALLCOLREF)) (TOK_WHERE (and (= (. (TOK_TABLE_OR_COL b) ds) '2008-04-08') (= (. (TOK_TABLE_OR_COL
b) hr) '14'))) (TOK_LIMIT 5)))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3
+  Stage-3 is a root stage
+  Stage-2 depends on stages: Stage-1, Stage-3
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery1:subq-subquery1:a 
+            Filter Operator
+              predicate:
+                  expr: ((ds = '2008-04-08') and (hr = '11'))
+                  type: boolean
+              Select Operator
+                expressions:
+                      expr: key
+                      type: string
+                      expr: value
+                      type: string
+                      expr: ds
+                      type: string
+                      expr: hr
+                      type: string
+                Reduce Output Operator
+                  key expressions:
+                        expr: 0
+                        type: string
+                  sort order: +
+                  tag: -1
+                  value expressions:
+                        expr: 0
+                        type: string
+                        expr: 1
+                        type: string
+                        expr: 2
+                        type: string
+                        expr: 3
+                        type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  name: binary_table
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+        /data/users/njain/hive5/hive5/build/ql/tmp/303379567/249444045.10002 
+          Union
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: string
+                    expr: 1
+                    type: string
+                    expr: 2
+                    type: string
+                    expr: 3
+                    type: string
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+        /data/users/njain/hive5/hive5/build/ql/tmp/303379567/249444045.10003 
+          Union
+            Select Operator
+              expressions:
+                    expr: 0
+                    type: string
+                    expr: 1
+                    type: string
+                    expr: 2
+                    type: string
+                    expr: 3
+                    type: string
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-3
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery2:subq-subquery2:b 
+            Filter Operator
+              predicate:
+                  expr: ((ds = '2008-04-08') and (hr = '14'))
+                  type: boolean
+              Select Operator
+                expressions:
+                      expr: key
+                      type: string
+                      expr: value
+                      type: string
+                      expr: ds
+                      type: string
+                      expr: hr
+                      type: string
+                Limit
+                  Reduce Output Operator
+                    sort order: 
+                    tag: -1
+                    value expressions:
+                          expr: 0
+                          type: string
+                          expr: 1
+                          type: string
+                          expr: 2
+                          type: string
+                          expr: 3
+                          type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  name: binary_table
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+query: select * from (
+  select * from srcpart a where a.ds = '2008-04-08' and a.hr = '11' order by a.key limit
5
+    union all
+  select * from srcpart b where b.ds = '2008-04-08' and b.hr = '14' limit 5
+)subq
+Input: default/srcpart/ds=2008-04-08/hr=11
+Output: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/11677377/90465736.10000
+0	val_0	2008-04-08	11
+0	val_0	2008-04-08	11
+0	val_0	2008-04-08	11
+10	val_10	2008-04-08	11
+100	val_100	2008-04-08	11

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup3.q.out?rev=773705&r1=773704&r2=773705&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup3.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/nullgroup3.q.out Mon May 11 22:00:28
2009
@@ -51,7 +51,7 @@
 query: select count(1) from tstparttbl
 Input: default/tstparttbl/ds=2008-04-09
 Input: default/tstparttbl/ds=2008-04-08
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/1221493083/197971283.10000
+Output: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/94234533/163827544.10000
 500
 query: DROP TABLE tstparttbl2
 query: CREATE TABLE tstparttbl2(KEY STRING, VALUE STRING) PARTITIONED BY(ds string) STORED
AS TEXTFILE
@@ -106,7 +106,8 @@
 query: select count(1) from tstparttbl2
 Input: default/tstparttbl2/ds=2008-04-09
 Input: default/tstparttbl2/ds=2008-04-08
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/200536084/27806968.10000
+Output: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/726239292/465785380.10000
+0
 query: DROP TABLE tstparttbl
 query: CREATE TABLE tstparttbl(KEY STRING, VALUE STRING) PARTITIONED BY(ds string) STORED
AS TEXTFILE
 query: LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE tstparttbl PARTITION (ds='2008-04-09')
@@ -160,7 +161,7 @@
 query: select count(1) from tstparttbl
 Input: default/tstparttbl/ds=2008-04-09
 Input: default/tstparttbl/ds=2008-04-08
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/1474931838/738960864.10000
+Output: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/115674074/219051439.10000
 500
 query: DROP TABLE tstparttbl2
 query: CREATE TABLE tstparttbl2(KEY STRING, VALUE STRING) PARTITIONED BY(ds string) STORED
AS TEXTFILE
@@ -215,4 +216,7 @@
 query: select count(1) from tstparttbl2
 Input: default/tstparttbl2/ds=2008-04-09
 Input: default/tstparttbl2/ds=2008-04-08
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/48371033/920895567.10000
+Output: /data/users/njain/hive5/hive5/ql/../build/ql/tmp/702539461/654313985.10000
+0
+query: DROP TABLE tstparttbl
+query: DROP TABLE tstparttbl2



Mime
View raw message