hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1561947 [11/17] - in /hive/branches/tez: ./ ant/ ant/src/org/apache/hadoop/hive/ant/ beeline/ cli/ cli/src/java/org/apache/hadoop/hive/cli/ common/ common/src/ common/src/java/org/apache/hadoop/hive/common/type/ common/src/java/org/apache/...
Date Tue, 28 Jan 2014 05:48:10 GMT
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java Tue Jan 28 05:48:03 2014
@@ -242,7 +242,7 @@ public class SimpleFetchOptimizer implem
 
       for (Partition partition : partsList.getNotDeniedPartns()) {
         inputs.add(new ReadEntity(partition));
-        listP.add(partition.getPartitionPath());
+        listP.add(partition.getDataLocation());
         partP.add(Utilities.getPartitionDesc(partition));
       }
       Table sourceTable = partsList.getSourceTable();
@@ -296,7 +296,7 @@ public class SimpleFetchOptimizer implem
       }
       long total = 0;
       for (Partition partition : partsList.getNotDeniedPartns()) {
-        Path path = partition.getPartitionPath();
+        Path path = partition.getDataLocation();
         total += getFileLength(jobConf, path, partition.getInputFormatClass());
       }
       return total;

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SizeBasedBigTableSelectorForAutoSMJ.java Tue Jan 28 05:48:03 2014
@@ -78,7 +78,7 @@ public abstract class SizeBasedBigTableS
   }
 
   protected long getSize(HiveConf conf, Partition partition) {
-    Path path = partition.getPartitionPath();
+    Path path = partition.getDataLocation();
     String size = partition.getParameters().get("totalSize");
 
     return getSize(conf, size, path);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java Tue Jan 28 05:48:03 2014
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.hadoop.hive.ql.optimizer;
 
 import java.util.ArrayList;
@@ -10,6 +27,7 @@ import java.util.Stack;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.Description;
@@ -30,6 +48,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
 import org.apache.hadoop.hive.ql.lib.Rule;
 import org.apache.hadoop.hive.ql.lib.RuleRegExp;
 import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.ParseContext;
@@ -42,12 +61,15 @@ import org.apache.hadoop.hive.ql.plan.Fe
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMin;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum;
 import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
 import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 
 /** There is a set of queries which can be answered entirely from statistics stored in metastore.
  * Examples of such queries are count(*), count(a), max(a), min(b) etc. Hive already collects
@@ -181,30 +203,29 @@ public class StatsOptimizer implements T
         Hive hive = Hive.get(pctx.getConf());
 
         for (AggregationDesc aggr : aggrs) {
-          if (aggr.getGenericUDAFName().equals(GenericUDAFCount.class.getAnnotation(
+          if (aggr.getGenericUDAFName().equals(GenericUDAFSum.class.getAnnotation(
+              Description.class).name())) {
+              if(!(aggr.getParameters().get(0) instanceof ExprNodeConstantDesc)){
+                return null;
+              }
+              Long rowCnt = getRowCnt(pctx, tsOp, tbl);
+              if(rowCnt == null) {
+                return null;
+              }
+              oneRow.add(HiveDecimal.create(((ExprNodeConstantDesc) aggr.getParameters().get(0))
+                .getValue().toString()).multiply(HiveDecimal.create(rowCnt)));
+              ois.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
+                PrimitiveCategory.DECIMAL));
+          }
+          else if (aggr.getGenericUDAFName().equals(GenericUDAFCount.class.getAnnotation(
               Description.class).name())) {
-            long rowCnt = 0;
+            Long rowCnt = 0L;
             if ((aggr.getParameters().isEmpty() || aggr.getParameters().get(0) instanceof
                 ExprNodeConstantDesc)) {
               // Its either count (*) or count(1) case
-              if(tbl.isPartitioned()) {
-                for (Partition part : hive.getAllPartitionsOf(tbl)) {
-                  long partRowCnt = Long.parseLong(part.getParameters()
-                    .get(StatsSetupConst.ROW_COUNT));
-                  if (partRowCnt < 1) {
-                    Log.debug("Partition doesn't have upto date stats " + part.getSpec());
-                    return null;
-                  }
-                  rowCnt += partRowCnt;
-                }
-              } else { // unpartitioned table
-                rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
-                if (rowCnt < 1) {
-                  // if rowCnt < 1 than its either empty table or table on which stats are not
-                  //  computed We assume the worse and don't attempt to optimize.
-                  Log.debug("Table doesn't have upto date stats " + tbl.getTableName());
-                  return null;
-                }
+              rowCnt = getRowCnt(pctx, tsOp, tbl);
+              if(rowCnt == null) {
+                return null;
               }
             } else {
               // Its count(col) case
@@ -221,7 +242,7 @@ public class StatsOptimizer implements T
                   Log.debug("Stats for table : " + tbl.getTableName() + " are not upto date.");
                   return null;
                   }
-            	rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
+                  rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
                 if (rowCnt < 1) {
                   Log.debug("Table doesn't have upto date stats " + tbl.getTableName());
                   return null;
@@ -238,13 +259,13 @@ public class StatsOptimizer implements T
                   rowCnt -= nullCnt;
                 }
               } else {
-                for (Partition part : hive.getAllPartitionsOf(tbl)) {
+                for (Partition part : pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions()) {
                   if (!StatsSetupConst.areStatsUptoDate(part.getParameters())) {
                     Log.debug("Stats for part : " + part.getSpec() + " are not upto date.");
                     return null;
                     }
-                	Long partRowCnt = Long.parseLong(part.getParameters()
-                    .get(StatsSetupConst.ROW_COUNT));
+                    Long partRowCnt = Long.parseLong(part.getParameters()
+                      .get(StatsSetupConst.ROW_COUNT));
                   if (partRowCnt < 1) {
                     Log.debug("Partition doesn't have upto date stats " + part.getSpec());
                     return null;
@@ -298,7 +319,7 @@ public class StatsOptimizer implements T
                 return null;
               }
             } else {
-              Set<Partition> parts = hive.getAllPartitionsOf(tbl);
+              Set<Partition> parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions();
               switch(type) {
               case Integeral: {
                 long maxVal = Long.MIN_VALUE;
@@ -370,7 +391,7 @@ public class StatsOptimizer implements T
                 return null;
               }
             } else {
-              Set<Partition> parts = hive.getAllPartitionsOf(tbl);
+              Set<Partition> parts = pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions();
               switch(type) {
               case Integeral: {
                 long minVal = Long.MAX_VALUE;
@@ -442,5 +463,28 @@ public class StatsOptimizer implements T
 
       return null;
     }
+
+    private Long getRowCnt (ParseContext pCtx, TableScanOperator tsOp, Table tbl) throws HiveException {
+        Long rowCnt = 0L;
+      if(tbl.isPartitioned()) {
+        for (Partition part : pctx.getPrunedPartitions(tsOp.getConf().getAlias(), tsOp).getPartitions()) {
+          long partRowCnt = Long.parseLong(part.getParameters().get(StatsSetupConst.ROW_COUNT));
+          if (partRowCnt < 1) {
+            Log.debug("Partition doesn't have upto date stats " + part.getSpec());
+            return null;
+          }
+          rowCnt += partRowCnt;
+        }
+      } else { // unpartitioned table
+        rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
+        if (rowCnt < 1) {
+          // if rowCnt < 1 than its either empty table or table on which stats are not
+          //  computed We assume the worse and don't attempt to optimize.
+          Log.debug("Table doesn't have upto date stats " + tbl.getTableName());
+          rowCnt = null;
+        }
+      }
+      return rowCnt;
+    }
   }
 }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java Tue Jan 28 05:48:03 2014
@@ -462,7 +462,7 @@ public class BucketingSortingOpProcFacto
       // Set the inferred bucket columns for the file this FileSink produces
       if (bucketCols != null) {
         List<BucketCol> newBucketCols = getNewBucketCols(bucketCols, colInfos);
-        bctx.getBucketedColsByDirectory().put(fop.getConf().getDirName(), newBucketCols);
+        bctx.getBucketedColsByDirectory().put(fop.getConf().getDirName().toString(), newBucketCols);
         bctx.setBucketedCols(fop, newBucketCols);
       }
 
@@ -471,7 +471,7 @@ public class BucketingSortingOpProcFacto
       // Set the inferred sort columns for the file this FileSink produces
       if (sortCols != null) {
         List<SortCol> newSortCols = getNewSortCols(sortCols, colInfos);
-        bctx.getSortedColsByDirectory().put(fop.getConf().getDirName(), newSortCols);
+        bctx.getSortedColsByDirectory().put(fop.getConf().getDirName().toString(), newSortCols);
         bctx.setSortedCols(fop, newSortCols);
       }
 

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java Tue Jan 28 05:48:03 2014
@@ -246,7 +246,7 @@ public class CommonJoinTaskDispatcher ex
     }
 
     // The mapJoinTaskFileSinkOperator writes to a different directory
-    String childMRPath = mapJoinTaskFileSinkOperator.getConf().getDirName();
+    String childMRPath = mapJoinTaskFileSinkOperator.getConf().getDirName().toString();
     List<String> childMRAliases = childMapWork.getPathToAliases().get(childMRPath);
     if (childMRAliases == null || childMRAliases.size() != 1) {
       return;
@@ -441,7 +441,7 @@ public class CommonJoinTaskDispatcher ex
 
       Configuration conf = context.getConf();
 
-      // If sizes of atleast n-1 tables in a n-way join is known, and their sum is smaller than
+      // If sizes of at least n-1 tables in a n-way join is known, and their sum is smaller than
       // the threshold size, convert the join into map-join and don't create a conditional task
       boolean convertJoinMapJoin = HiveConf.getBoolVar(conf,
           HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASK);
@@ -451,47 +451,32 @@ public class CommonJoinTaskDispatcher ex
         long mapJoinSize = HiveConf.getLongVar(conf,
             HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD);
 
-        boolean bigTableFound = false;
-        long largestBigTableCandidateSize = -1;
-        long sumTableSizes = 0;
-        for (String alias : aliasToWork.keySet()) {
+        Long bigTableSize = null;
+        Set<String> aliases = aliasToWork.keySet();
+        for (String alias : aliases) {
           int tablePosition = getPosition(currWork, joinOp, alias);
-          boolean bigTableCandidate = bigTableCandidates.contains(tablePosition);
-          Long size = aliasToSize.get(alias);
-          // The size is not available at compile time if the input is a sub-query.
-          // If the size of atleast n-1 inputs for a n-way join are available at compile time,
-          // and the sum of them is less than the specified threshold, then convert the join
-          // into a map-join without the conditional task.
-          if ((size == null) || (size > mapJoinSize)) {
-            sumTableSizes += largestBigTableCandidateSize;
-            if (bigTableFound || (sumTableSizes > mapJoinSize) || !bigTableCandidate) {
-              convertJoinMapJoin = false;
-              break;
-            }
-            bigTableFound = true;
+          if (!bigTableCandidates.contains(tablePosition)) {
+            continue;
+          }
+          long sumOfOthers = Utilities.sumOfExcept(aliasToSize, aliases, alias);
+          if (sumOfOthers < 0 || sumOfOthers > mapJoinSize) {
+            continue; // some small alias is not known or too big
+          }
+          if (bigTableSize == null && bigTablePosition >= 0 && tablePosition < bigTablePosition) {
+            continue; // prefer right most alias
+          }
+          Long aliasSize = aliasToSize.get(alias);
+          if (bigTableSize == null || (aliasSize != null && aliasSize > bigTableSize)) {
             bigTablePosition = tablePosition;
-            largestBigTableCandidateSize = mapJoinSize + 1;
-          } else {
-            if (bigTableCandidate && size > largestBigTableCandidateSize) {
-              bigTablePosition = tablePosition;
-              sumTableSizes += largestBigTableCandidateSize;
-              largestBigTableCandidateSize = size;
-            } else {
-              sumTableSizes += size;
-            }
-            if (sumTableSizes > mapJoinSize) {
-              convertJoinMapJoin = false;
-              break;
-            }
+            bigTableSize = aliasSize;
           }
         }
       }
 
-      String bigTableAlias = null;
       currWork.setOpParseCtxMap(parseCtx.getOpParseCtx());
       currWork.setJoinTree(joinTree);
 
-      if (convertJoinMapJoin) {
+      if (bigTablePosition >= 0) {
         // create map join task and set big table as bigTablePosition
         MapRedTask newTask = convertTaskToMapJoinTask(currTask.getWork(), bigTablePosition).getFirst();
 
@@ -521,7 +506,7 @@ public class CommonJoinTaskDispatcher ex
         // create map join task and set big table as i
         ObjectPair<MapRedTask, String> newTaskAlias = convertTaskToMapJoinTask(newWork, i);
         MapRedTask newTask = newTaskAlias.getFirst();
-        bigTableAlias = newTaskAlias.getSecond();
+        String bigTableAlias = newTaskAlias.getSecond();
 
         if (cannotConvert(bigTableAlias, aliasToSize,
             aliasTotalKnownInputSize, ThresholdOfSmallTblSizeSum)) {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java Tue Jan 28 05:48:03 2014
@@ -123,21 +123,20 @@ public final class GenMRSkewJoinProcesso
     Task<? extends Serializable> child =
         children != null && children.size() == 1 ? children.get(0) : null;
 
-    String baseTmpDir = parseCtx.getContext().getMRTmpFileURI();
+    Path baseTmpDir = parseCtx.getContext().getMRTmpPath();
 
     JoinDesc joinDescriptor = joinOp.getConf();
     Map<Byte, List<ExprNodeDesc>> joinValues = joinDescriptor.getExprs();
     int numAliases = joinValues.size();
 
-    Map<Byte, String> bigKeysDirMap = new HashMap<Byte, String>();
-    Map<Byte, Map<Byte, String>> smallKeysDirMap = new HashMap<Byte, Map<Byte, String>>();
-    Map<Byte, String> skewJoinJobResultsDir = new HashMap<Byte, String>();
+    Map<Byte, Path> bigKeysDirMap = new HashMap<Byte, Path>();
+    Map<Byte, Map<Byte, Path>> smallKeysDirMap = new HashMap<Byte, Map<Byte, Path>>();
+    Map<Byte, Path> skewJoinJobResultsDir = new HashMap<Byte, Path>();
     Byte[] tags = joinDescriptor.getTagOrder();
     for (int i = 0; i < numAliases; i++) {
       Byte alias = tags[i];
-      String bigKeysDir = getBigKeysDir(baseTmpDir, alias);
-      bigKeysDirMap.put(alias, bigKeysDir);
-      Map<Byte, String> smallKeysMap = new HashMap<Byte, String>();
+      bigKeysDirMap.put(alias, getBigKeysDir(baseTmpDir, alias));
+      Map<Byte, Path> smallKeysMap = new HashMap<Byte, Path>();
       smallKeysDirMap.put(alias, smallKeysMap);
       for (Byte src2 : tags) {
         if (!src2.equals(alias)) {
@@ -154,8 +153,8 @@ public final class GenMRSkewJoinProcesso
     joinDescriptor.setSkewKeyDefinition(HiveConf.getIntVar(parseCtx.getConf(),
         HiveConf.ConfVars.HIVESKEWJOINKEY));
 
-    HashMap<String, Task<? extends Serializable>> bigKeysDirToTaskMap =
-      new HashMap<String, Task<? extends Serializable>>();
+    HashMap<Path, Task<? extends Serializable>> bigKeysDirToTaskMap =
+      new HashMap<Path, Task<? extends Serializable>>();
     List<Serializable> listWorks = new ArrayList<Serializable>();
     List<Task<? extends Serializable>> listTasks = new ArrayList<Task<? extends Serializable>>();
     MapredWork currPlan = (MapredWork) currTask.getWork();
@@ -272,13 +271,13 @@ public final class GenMRSkewJoinProcesso
       ArrayList<String> aliases = new ArrayList<String>();
       String alias = src.toString();
       aliases.add(alias);
-      String bigKeyDirPath = bigKeysDirMap.get(src);
-      newPlan.getPathToAliases().put(bigKeyDirPath, aliases);
+      Path bigKeyDirPath = bigKeysDirMap.get(src);
+      newPlan.getPathToAliases().put(bigKeyDirPath.toString(), aliases);
 
       newPlan.getAliasToWork().put(alias, tblScan_op);
       PartitionDesc part = new PartitionDesc(tableDescList.get(src), null);
 
-      newPlan.getPathToPartitionInfo().put(bigKeyDirPath, part);
+      newPlan.getPathToPartitionInfo().put(bigKeyDirPath.toString(), part);
       newPlan.getAliasToPartnInfo().put(alias, part);
 
       Operator<? extends OperatorDesc> reducer = clonePlan.getReduceWork().getReducer();
@@ -297,7 +296,7 @@ public final class GenMRSkewJoinProcesso
       MapredLocalWork localPlan = new MapredLocalWork(
           new LinkedHashMap<String, Operator<? extends OperatorDesc>>(),
           new LinkedHashMap<String, FetchWork>());
-      Map<Byte, String> smallTblDirs = smallKeysDirMap.get(src);
+      Map<Byte, Path> smallTblDirs = smallKeysDirMap.get(src);
 
       for (int j = 0; j < numAliases; j++) {
         if (j == i) {
@@ -306,7 +305,7 @@ public final class GenMRSkewJoinProcesso
         Byte small_alias = tags[j];
         Operator<? extends OperatorDesc> tblScan_op2 = parentOps[j];
         localPlan.getAliasToWork().put(small_alias.toString(), tblScan_op2);
-        Path tblDir = new Path(smallTblDirs.get(small_alias));
+        Path tblDir = smallTblDirs.get(small_alias);
         localPlan.getAliasToFetchWork().put(small_alias.toString(),
             new FetchWork(tblDir, tableDescList.get(small_alias)));
       }
@@ -393,20 +392,19 @@ public final class GenMRSkewJoinProcesso
   private static String SMALLKEYS = "smallkeys";
   private static String RESULTS = "results";
 
-  static String getBigKeysDir(String baseDir, Byte srcTbl) {
-    return baseDir + Path.SEPARATOR + skewJoinPrefix + UNDERLINE + BIGKEYS
-        + UNDERLINE + srcTbl;
+  static Path getBigKeysDir(Path baseDir, Byte srcTbl) {
+    return new Path(baseDir, skewJoinPrefix + UNDERLINE + BIGKEYS + UNDERLINE + srcTbl);
   }
 
-  static String getBigKeysSkewJoinResultDir(String baseDir, Byte srcTbl) {
-    return baseDir + Path.SEPARATOR + skewJoinPrefix + UNDERLINE + BIGKEYS
-        + UNDERLINE + RESULTS + UNDERLINE + srcTbl;
+  static Path getBigKeysSkewJoinResultDir(Path baseDir, Byte srcTbl) {
+    return new Path(baseDir, skewJoinPrefix + UNDERLINE + BIGKEYS
+        + UNDERLINE + RESULTS + UNDERLINE + srcTbl);
   }
 
-  static String getSmallKeysDir(String baseDir, Byte srcTblBigTbl,
+  static Path getSmallKeysDir(Path baseDir, Byte srcTblBigTbl,
       Byte srcTblSmallTbl) {
-    return baseDir + Path.SEPARATOR + skewJoinPrefix + UNDERLINE + SMALLKEYS
-        + UNDERLINE + srcTblBigTbl + UNDERLINE + srcTblSmallTbl;
+    return new Path(baseDir, skewJoinPrefix + UNDERLINE + SMALLKEYS
+        + UNDERLINE + srcTblBigTbl + UNDERLINE + srcTblSmallTbl);
   }
 
 }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java Tue Jan 28 05:48:03 2014
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Stack;
 
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.exec.ConditionalTask;
 import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
@@ -101,10 +102,10 @@ public class MapJoinResolver implements 
       if (localwork != null) {
         // get the context info and set up the shared tmp URI
         Context ctx = physicalContext.getContext();
-        String tmpFileURI = Utilities.generateTmpURI(ctx.getLocalTmpFileURI(), currTask.getId());
-        localwork.setTmpFileURI(tmpFileURI);
-        String hdfsTmpURI = Utilities.generateTmpURI(ctx.getMRTmpFileURI(), currTask.getId());
-        mapredWork.getMapWork().setTmpHDFSFileURI(hdfsTmpURI);
+        Path tmpPath = Utilities.generateTmpPath(ctx.getLocalTmpPath(), currTask.getId());
+        localwork.setTmpPath(tmpPath);
+        mapredWork.getMapWork().setTmpHDFSPath(Utilities.generateTmpPath(
+          ctx.getMRTmpPath(), currTask.getId()));
         // create a task for this local work; right now, this local work is shared
         // by the original MapredTask and this new generated MapredLocalTask.
         MapredLocalTask localTask = (MapredLocalTask) TaskFactory.get(localwork, physicalContext
@@ -130,7 +131,7 @@ public class MapJoinResolver implements 
         // create new local work and setup the dummy ops
         MapredLocalWork newLocalWork = new MapredLocalWork();
         newLocalWork.setDummyParentOp(dummyOps);
-        newLocalWork.setTmpFileURI(tmpFileURI);
+        newLocalWork.setTmpPath(tmpPath);
         newLocalWork.setInputFileChangeSensitive(localwork.getInputFileChangeSensitive());
         newLocalWork.setBucketMapjoinContext(localwork.copyPartSpecMappingOnly());
         mapredWork.getMapWork().setMapLocalWork(newLocalWork);
@@ -166,15 +167,15 @@ public class MapJoinResolver implements 
               // get bigKeysDirToTaskMap
               ConditionalResolverSkewJoinCtx context = (ConditionalResolverSkewJoinCtx) conditionalTask
                   .getResolverCtx();
-              HashMap<String, Task<? extends Serializable>> bigKeysDirToTaskMap = context
+              HashMap<Path, Task<? extends Serializable>> bigKeysDirToTaskMap = context
                   .getDirToTaskMap();
               // to avoid concurrent modify the hashmap
-              HashMap<String, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<String, Task<? extends Serializable>>();
+              HashMap<Path, Task<? extends Serializable>> newbigKeysDirToTaskMap = new HashMap<Path, Task<? extends Serializable>>();
               // reset the resolver
-              for (Map.Entry<String, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
+              for (Map.Entry<Path, Task<? extends Serializable>> entry : bigKeysDirToTaskMap
                   .entrySet()) {
                 Task<? extends Serializable> task = entry.getValue();
-                String key = entry.getKey();
+                Path key = entry.getKey();
                 if (task.equals(currTask)) {
                   newbigKeysDirToTaskMap.put(key, localTask);
                 } else {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java Tue Jan 28 05:48:03 2014
@@ -238,7 +238,7 @@ public class MetadataOnlyOptimizer imple
       for (String path : paths) {
         PartitionDesc partDesc = work.getPathToPartitionInfo().get(path);
         PartitionDesc newPartition = changePartitionToMetadataOnly(partDesc);
-        Path fakePath = new Path(physicalContext.getContext().getMRTmpFileURI()
+        Path fakePath = new Path(physicalContext.getContext().getMRTmpPath()
             + newPartition.getTableName()
             + encode(newPartition.getPartSpec()));
         work.getPathToPartitionInfo().remove(path);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Tue Jan 28 05:48:03 2014
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.exec.Ma
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
 import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
 import org.apache.hadoop.hive.ql.exec.SelectOperator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Task;
@@ -78,6 +79,7 @@ import org.apache.hadoop.hive.ql.plan.Ma
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
 import org.apache.hadoop.hive.ql.plan.TableScanDesc;
 import org.apache.hadoop.hive.ql.plan.TezWork;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
@@ -555,6 +557,8 @@ public class Vectorizer implements Physi
       case MAPJOIN:
         if (op instanceof MapJoinOperator) {
           ret = validateMapJoinOperator((MapJoinOperator) op);
+        } else if (op instanceof SMBMapJoinOperator) {
+          ret = validateSMBMapJoinOperator((SMBMapJoinOperator) op);
         }
         break;
       case GROUPBY:
@@ -583,6 +587,12 @@ public class Vectorizer implements Physi
     return ret;
   }
 
+  private boolean validateSMBMapJoinOperator(SMBMapJoinOperator op) {
+    SMBJoinDesc desc = op.getConf();
+    // Validation is the same as for map join, since the 'small' tables are not vectorized
+    return validateMapJoinDesc(desc);
+  }
+
   private boolean validateTableScanOperator(TableScanOperator op) {
     TableScanDesc desc = op.getConf();
     return !desc.isGatherStats();
@@ -590,6 +600,10 @@ public class Vectorizer implements Physi
 
   private boolean validateMapJoinOperator(MapJoinOperator op) {
     MapJoinDesc desc = op.getConf();
+    return validateMapJoinDesc(desc);
+  }
+  
+  private boolean validateMapJoinDesc(MapJoinDesc desc) {
     byte posBigTable = (byte) desc.getPosBigTable();
     List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable);
     List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java Tue Jan 28 05:48:03 2014
@@ -212,7 +212,7 @@ public final class UnionProcFactory {
         // for each sub-query. Also, these different filesinks need to be linked to each other
         FileSinkOperator fileSinkOp = (FileSinkOperator)stack.get(pos);
         // For file sink operator, change the directory name
-        String parentDirName = fileSinkOp.getConf().getDirName();
+        Path parentDirName = fileSinkOp.getConf().getDirName();
 
         // Clone the fileSinkDesc of the final fileSink and create similar fileSinks at
         // each parent
@@ -220,9 +220,7 @@ public final class UnionProcFactory {
 
         for (Operator<? extends OperatorDesc> parent : parents) {
           FileSinkDesc fileSinkDesc = (FileSinkDesc) fileSinkOp.getConf().clone();
-
-          String dirName = parentDirName + Path.SEPARATOR + parent.getIdentifier() ;
-          fileSinkDesc.setDirName(dirName);
+          fileSinkDesc.setDirName(new Path(parentDirName, parent.getIdentifier()));
           fileSinkDesc.setLinkedFileSink(true);
           fileSinkDesc.setParentDir(parentDirName);
           parent.setChildOperators(null);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -39,6 +39,7 @@ import org.antlr.runtime.tree.Tree;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
@@ -785,7 +786,7 @@ public abstract class BaseSemanticAnalyz
         }
 
         // check if the columns, as well as value types in the partition() clause are valid
-        validatePartSpec(tableHandle, tmpPartSpec, ast, conf);
+        validatePartSpec(tableHandle, tmpPartSpec, ast, conf, false);
 
         List<FieldSchema> parts = tableHandle.getPartitionKeys();
         partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
@@ -1187,8 +1188,8 @@ public abstract class BaseSemanticAnalyz
   }
 
   public static void validatePartSpec(Table tbl, Map<String, String> partSpec,
-      ASTNode astNode, HiveConf conf) throws SemanticException {
-    Utilities.validatePartSpecColumnNames(tbl, partSpec);
+      ASTNode astNode, HiveConf conf, boolean shouldBeFull) throws SemanticException {
+    tbl.validatePartColumnNames(partSpec, shouldBeFull);
 
     if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TYPE_CHECK_ON_INSERT)) {
       return;
@@ -1256,9 +1257,6 @@ public abstract class BaseSemanticAnalyz
     }
   }
 
-  /** A fixed date format to be used for hive partition column values. */
-  private static final DateFormat partitionDateFormat = new SimpleDateFormat("yyyy-MM-dd");
-
   private static String normalizeDateCol(
       Object colValue, String originalColSpec) throws SemanticException {
     Date value;
@@ -1269,7 +1267,7 @@ public abstract class BaseSemanticAnalyz
     } else {
       throw new SemanticException("Unexpected date type " + colValue.getClass());
     }
-    return partitionDateFormat.format(value);
+    return HiveMetaStore.PARTITION_DATE_FORMAT.format(value);
   }
 
   protected Database getDatabase(String dbName) throws SemanticException {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -233,11 +233,11 @@ public class DDLSemanticAnalyzer extends
       ast = (ASTNode) ast.getChild(1);
       if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_FILEFORMAT) {
         analyzeAlterTableFileFormat(ast, tableName, partSpec);
-      } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_ALTERPARTS_PROTECTMODE) {
+      } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_PROTECTMODE) {
         analyzeAlterTableProtectMode(ast, tableName, partSpec);
       } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_LOCATION) {
         analyzeAlterTableLocation(ast, tableName, partSpec);
-      } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_ALTERPARTS_MERGEFILES) {
+      } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_MERGEFILES) {
         analyzeAlterTablePartMergeFiles(tablePart, ast, tableName, partSpec);
       } else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_SERIALIZER) {
         analyzeAlterTableSerde(ast, tableName, partSpec);
@@ -267,51 +267,51 @@ public class DDLSemanticAnalyzer extends
       analyzeDropIndex(ast);
       break;
     case HiveParser.TOK_DESCTABLE:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeDescribeTable(ast);
       break;
     case HiveParser.TOK_SHOWDATABASES:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowDatabases(ast);
       break;
     case HiveParser.TOK_SHOWTABLES:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowTables(ast);
       break;
     case HiveParser.TOK_SHOWCOLUMNS:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowColumns(ast);
       break;
     case HiveParser.TOK_SHOW_TABLESTATUS:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowTableStatus(ast);
       break;
     case HiveParser.TOK_SHOW_TBLPROPERTIES:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowTableProperties(ast);
       break;
     case HiveParser.TOK_SHOWFUNCTIONS:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowFunctions(ast);
       break;
     case HiveParser.TOK_SHOWLOCKS:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowLocks(ast);
       break;
     case HiveParser.TOK_SHOWDBLOCKS:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowDbLocks(ast);
       break;
     case HiveParser.TOK_DESCFUNCTION:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeDescFunction(ast);
       break;
     case HiveParser.TOK_DESCDATABASE:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeDescDatabase(ast);
       break;
     case HiveParser.TOK_MSCK:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeMetastoreCheck(ast);
       break;
     case HiveParser.TOK_DROPVIEW:
@@ -365,8 +365,8 @@ public class DDLSemanticAnalyzer extends
     case HiveParser.TOK_ALTERTABLE_DROPPARTS:
       analyzeAlterTableDropParts(ast, false);
       break;
-    case HiveParser.TOK_ALTERTABLE_ALTERPARTS:
-      analyzeAlterTableAlterParts(ast);
+    case HiveParser.TOK_ALTERTABLE_PARTCOLTYPE:
+      analyzeAlterTablePartColType(ast);
       break;
     case HiveParser.TOK_ALTERTABLE_PROPERTIES:
       analyzeAlterTableProps(ast, false, false);
@@ -381,15 +381,15 @@ public class DDLSemanticAnalyzer extends
       analyzeAlterIndexProps(ast);
       break;
     case HiveParser.TOK_SHOWPARTITIONS:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowPartitions(ast);
       break;
     case HiveParser.TOK_SHOW_CREATETABLE:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowCreateTable(ast);
       break;
     case HiveParser.TOK_SHOWINDEXES:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowIndexes(ast);
       break;
     case HiveParser.TOK_LOCKTABLE:
@@ -423,9 +423,13 @@ public class DDLSemanticAnalyzer extends
       analyzeDropRole(ast);
       break;
     case HiveParser.TOK_SHOW_ROLE_GRANT:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowRoleGrant(ast);
       break;
+    case HiveParser.TOK_SHOW_ROLES:
+      ctx.setResFile(ctx.getLocalTmpPath());
+      analyzeShowRoles(ast);
+      break;
     case HiveParser.TOK_GRANT_ROLE:
       analyzeGrantRevokeRole(true, ast);
       break;
@@ -436,7 +440,7 @@ public class DDLSemanticAnalyzer extends
       analyzeGrant(ast);
       break;
     case HiveParser.TOK_SHOW_GRANT:
-      ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+      ctx.setResFile(ctx.getLocalTmpPath());
       analyzeShowGrant(ast);
       break;
     case HiveParser.TOK_REVOKE:
@@ -515,6 +519,15 @@ public class DDLSemanticAnalyzer extends
     }
   }
 
+  private void analyzeShowRoles(ASTNode ast) {
+    RoleDDLDesc showRolesDesc = new RoleDDLDesc(null, null,
+        RoleDDLDesc.RoleOperation.SHOW_ROLES, null);
+    showRolesDesc.setResFile(ctx.getResFile().toString());
+    rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
+        showRolesDesc), conf));
+    setFetchTask(createFetchTask(RoleDDLDesc.getSchema()));
+  }
+
   private void analyzeAlterDatabase(ASTNode ast) throws SemanticException {
 
     String dbName = unescapeIdentifier(ast.getChild(0).getText());
@@ -758,7 +771,7 @@ public class DDLSemanticAnalyzer extends
           Partition part = db.getPartition(table, partSpec, false);
 
           Path tabPath = table.getPath();
-          Path partPath = part.getPartitionPath();
+          Path partPath = part.getDataLocation();
 
           // if the table is in a different dfs than the partition,
           // replace the partition's dfs with the table's dfs.
@@ -840,9 +853,9 @@ public class DDLSemanticAnalyzer extends
         TableDesc tblDesc = Utilities.getTableDesc(table);
         // Write the output to temporary directory and move it to the final location at the end
         // so the operation is atomic.
-        String queryTmpdir = ctx.getExternalTmpFileURI(newTblPartLoc.toUri());
-        truncateTblDesc.setOutputDir(new Path(queryTmpdir));
-        LoadTableDesc ltd = new LoadTableDesc(new Path(queryTmpdir), tblDesc,
+        Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc.toUri());
+        truncateTblDesc.setOutputDir(queryTmpdir);
+        LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc,
             partSpec == null ? new HashMap<String, String>() : partSpec);
         ltd.setLbCtx(lbCtx);
         Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
@@ -1402,7 +1415,7 @@ public class DDLSemanticAnalyzer extends
           isArchived = ArchiveUtils.isArchived(part);
 
           Path tabPath = tblObj.getPath();
-          Path partPath = part.getPartitionPath();
+          Path partPath = part.getDataLocation();
 
           // if the table is in a different dfs than the partition,
           // replace the partition's dfs with the table's dfs.
@@ -1455,9 +1468,9 @@ public class DDLSemanticAnalyzer extends
       ddlWork.setNeedLock(true);
       Task<? extends Serializable> mergeTask = TaskFactory.get(ddlWork, conf);
       TableDesc tblDesc = Utilities.getTableDesc(tblObj);
-      String queryTmpdir = ctx.getExternalTmpFileURI(newTblPartLoc.toUri());
-      mergeDesc.setOutputDir(new Path(queryTmpdir));
-      LoadTableDesc ltd = new LoadTableDesc(new Path(queryTmpdir), tblDesc,
+      Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc.toUri());
+      mergeDesc.setOutputDir(queryTmpdir);
+      LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc,
           partSpec == null ? new HashMap<String, String>() : partSpec);
       ltd.setLbCtx(lbCtx);
       Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
@@ -2420,7 +2433,7 @@ public class DDLSemanticAnalyzer extends
         dropTblDesc), conf));
   }
 
-  private void analyzeAlterTableAlterParts(ASTNode ast)
+  private void analyzeAlterTablePartColType(ASTNode ast)
       throws SemanticException {
     // get table name
     String tblName = getUnescapedName((ASTNode)ast.getChild(0));
@@ -2498,32 +2511,30 @@ public class DDLSemanticAnalyzer extends
     validateAlterTableType(tab, AlterTableTypes.ADDPARTITION, expectView);
     inputs.add(new ReadEntity(tab));
 
-    List<AddPartitionDesc> partitionDescs = new ArrayList<AddPartitionDesc>();
-
     int numCh = ast.getChildCount();
     int start = ifNotExists ? 2 : 1;
 
     String currentLocation = null;
     Map<String, String> currentPart = null;
+    // Parser has done some verification, so the order of tokens doesn't need to be verified here.
+    AddPartitionDesc addPartitionDesc = new AddPartitionDesc(tab.getDbName(), tblName, ifNotExists);
     for (int num = start; num < numCh; num++) {
       ASTNode child = (ASTNode) ast.getChild(num);
       switch (child.getToken().getType()) {
       case HiveParser.TOK_PARTSPEC:
         if (currentPart != null) {
-          Partition partition = getPartitionForOutput(tab, currentPart);
-          if (partition == null || !ifNotExists) {
-            AddPartitionDesc addPartitionDesc = new AddPartitionDesc(
-              tab.getDbName(), tblName, currentPart,
-              currentLocation, ifNotExists, expectView);
-            partitionDescs.add(addPartitionDesc);
-          }
+          addPartitionDesc.addPartition(currentPart, currentLocation);
           currentLocation = null;
         }
         currentPart = getPartSpec(child);
-        validatePartSpec(tab, currentPart, (ASTNode)child, conf);
+        validatePartitionValues(currentPart); // validate reserved values
+        validatePartSpec(tab, currentPart, (ASTNode)child, conf, true);
         break;
       case HiveParser.TOK_PARTITIONLOCATION:
         // if location specified, set in partition
+        if (isView) {
+          throw new SemanticException("LOCATION clause illegal for view partition");
+        }
         currentLocation = unescapeSQLString(child.getChild(0).getText());
         break;
       default:
@@ -2533,47 +2544,25 @@ public class DDLSemanticAnalyzer extends
 
     // add the last one
     if (currentPart != null) {
-      Partition partition = getPartitionForOutput(tab, currentPart);
-      if (partition == null || !ifNotExists) {
-        AddPartitionDesc addPartitionDesc = new AddPartitionDesc(
-          tab.getDbName(), tblName, currentPart,
-          currentLocation, ifNotExists, expectView);
-        partitionDescs.add(addPartitionDesc);
-      }
+      addPartitionDesc.addPartition(currentPart, currentLocation);
     }
 
-    if (partitionDescs.isEmpty()) {
+    if (addPartitionDesc.getPartitionCount() == 0) {
       // nothing to do
       return;
     }
 
-    for (AddPartitionDesc addPartitionDesc : partitionDescs) {
-      try {
-        tab.isValidSpec(addPartitionDesc.getPartSpec());
-      } catch (HiveException ex) {
-        throw new SemanticException(ErrorMsg.INVALID_PARTITION_SPEC.getMsg(ex.getMessage()));
-      }
-      rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
-          addPartitionDesc), conf));
-    }
+    rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), addPartitionDesc), conf));
 
     if (isView) {
-      // Compile internal query to capture underlying table partition
-      // dependencies
+      // Compile internal query to capture underlying table partition dependencies
       StringBuilder cmd = new StringBuilder();
       cmd.append("SELECT * FROM ");
       cmd.append(HiveUtils.unparseIdentifier(tblName));
       cmd.append(" WHERE ");
       boolean firstOr = true;
-      for (AddPartitionDesc partitionDesc : partitionDescs) {
-        // Perform this check early so that we get a better error message.
-        try {
-          // Note that isValidSpec throws an exception (it never
-          // actually returns false).
-          tab.isValidSpec(partitionDesc.getPartSpec());
-        } catch (HiveException ex) {
-          throw new SemanticException(ErrorMsg.INVALID_PARTITION_SPEC.getMsg(ex.getMessage()));
-        }
+      for (int i = 0; i < addPartitionDesc.getPartitionCount(); ++i) {
+        AddPartitionDesc.OnePartitionDesc partitionDesc = addPartitionDesc.getPartition(i);
         if (firstOr) {
           firstOr = false;
         } else {
@@ -2581,8 +2570,7 @@ public class DDLSemanticAnalyzer extends
         }
         boolean firstAnd = true;
         cmd.append("(");
-        for (Map.Entry<String, String> entry : partitionDesc.getPartSpec().entrySet())
-        {
+        for (Map.Entry<String, String> entry : partitionDesc.getPartSpec().entrySet()) {
           if (firstAnd) {
             firstAnd = false;
           } else {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -22,7 +22,6 @@ import java.io.Serializable;
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.exec.Task;
@@ -65,7 +64,7 @@ public class ExplainSemanticAnalyzer ext
     sem.analyze(input, ctx);
     sem.validate();
 
-    ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
+    ctx.setResFile(ctx.getLocalTmpPath());
     List<Task<? extends Serializable>> tasks = sem.getRootTasks();
     Task<? extends Serializable> fetchTask = sem.getFetchTask();
     if (tasks == null) {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -87,8 +87,7 @@ public class ExportSemanticAnalyzer exte
       if (ts.tableHandle.isPartitioned()) {
         partitions = (ts.partitions != null) ? ts.partitions : db.getPartitions(ts.tableHandle);
       }
-      String tmpfile = ctx.getLocalTmpFileURI();
-      Path path = new Path(tmpfile, "_metadata");
+      Path path = new Path(ctx.getLocalTmpPath(), "_metadata");
       EximUtil.createExportDump(FileSystem.getLocal(conf), path, ts.tableHandle, partitions);
       Task<? extends Serializable> rTask = TaskFactory.get(new CopyWork(
           path, new Path(toURI), false), conf);
@@ -105,19 +104,19 @@ public class ExportSemanticAnalyzer exte
 
     if (ts.tableHandle.isPartitioned()) {
       for (Partition partition : partitions) {
-        URI fromURI = partition.getDataLocation();
+        Path fromPath = partition.getDataLocation();
         Path toPartPath = new Path(parentPath, partition.getName());
         Task<? extends Serializable> rTask = TaskFactory.get(
-            new CopyWork(new Path(fromURI), toPartPath, false),
+            new CopyWork(fromPath, toPartPath, false),
             conf);
         rootTasks.add(rTask);
         inputs.add(new ReadEntity(partition));
       }
     } else {
-      URI fromURI = ts.tableHandle.getDataLocation();
+      Path fromPath = ts.tableHandle.getDataLocation();
       Path toDataPath = new Path(parentPath, "data");
       Task<? extends Serializable> rTask = TaskFactory.get(new CopyWork(
-          new Path(fromURI), toDataPath, false), conf);
+          fromPath, toDataPath, false), conf);
       rootTasks.add(rTask);
       inputs.add(new ReadEntity(ts.tableHandle));
     }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g Tue Jan 28 05:48:03 2014
@@ -204,7 +204,7 @@ subQuerySource
 @init { gParent.msgs.push("subquery source"); }
 @after { gParent.msgs.pop(); }
     :
-    LPAREN queryStatementExpression RPAREN identifier -> ^(TOK_SUBQUERY queryStatementExpression identifier)
+    LPAREN queryStatementExpression[false] RPAREN identifier -> ^(TOK_SUBQUERY queryStatementExpression identifier)
     ;
 
 //---------------------- Rules for parsing PTF clauses -----------------------------

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Tue Jan 28 05:48:03 2014
@@ -280,6 +280,7 @@ KW_NOSCAN: 'NOSCAN';
 KW_PARTIALSCAN: 'PARTIALSCAN';
 KW_USER: 'USER';
 KW_ROLE: 'ROLE';
+KW_ROLES: 'ROLES';
 KW_INNER: 'INNER';
 KW_EXCHANGE: 'EXCHANGE';
 KW_ADMIN: 'ADMIN';

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Jan 28 05:48:03 2014
@@ -138,8 +138,9 @@ TOK_ALTERTABLE_RENAMEPART;
 TOK_ALTERTABLE_REPLACECOLS;
 TOK_ALTERTABLE_ADDPARTS;
 TOK_ALTERTABLE_DROPPARTS;
-TOK_ALTERTABLE_ALTERPARTS;
-TOK_ALTERTABLE_ALTERPARTS_PROTECTMODE;
+TOK_ALTERTABLE_PARTCOLTYPE;
+TOK_ALTERTABLE_PROTECTMODE;
+TOK_ALTERTABLE_MERGEFILES;
 TOK_ALTERTABLE_TOUCH;
 TOK_ALTERTABLE_ARCHIVE;
 TOK_ALTERTABLE_UNARCHIVE;
@@ -276,6 +277,7 @@ TOK_PRIV_OBJECT_COL;
 TOK_GRANT_ROLE;
 TOK_REVOKE_ROLE;
 TOK_SHOW_ROLE_GRANT;
+TOK_SHOW_ROLES;
 TOK_SHOWINDEXES;
 TOK_SHOWDBLOCKS;
 TOK_INDEXCOMMENT;
@@ -284,7 +286,6 @@ TOK_DATABASEPROPERTIES;
 TOK_DATABASELOCATION;
 TOK_DBPROPLIST;
 TOK_ALTERDATABASE_PROPERTIES;
-TOK_ALTERTABLE_ALTERPARTS_MERGEFILES;
 TOK_TABNAME;
 TOK_TABSRC;
 TOK_RESTRICT;
@@ -385,7 +386,7 @@ import java.util.HashMap;
     xlateMap.put("KW_ALTER", "ALTER");
     xlateMap.put("KW_DESCRIBE", "DESCRIBE");
     xlateMap.put("KW_DROP", "DROP");
-    xlateMap.put("KW_REANME", "REANME");
+    xlateMap.put("KW_RENAME", "RENAME");
     xlateMap.put("KW_TO", "TO");
     xlateMap.put("KW_COMMENT", "COMMENT");
     xlateMap.put("KW_BOOLEAN", "BOOLEAN");
@@ -555,6 +556,13 @@ import java.util.HashMap;
     }
     return msg;
   }
+  
+  // counter to generate unique union aliases
+  private int aliasCounter;
+  
+  private String generateUnionAlias() {
+    return "_u" + (++aliasCounter);
+  }
 }
 
 @rulecatch {
@@ -580,7 +588,7 @@ explainStatement
 execStatement
 @init { msgs.push("statement"); }
 @after { msgs.pop(); }
-    : queryStatementExpression
+    : queryStatementExpression[true]
     | loadStatement
     | exportStatement
     | importStatement
@@ -640,6 +648,7 @@ ddlStatement
     | revokePrivileges
     | showGrants
     | showRoleGrants
+    | showRoles
     | grantRole
     | revokeRole
     ;
@@ -759,7 +768,7 @@ createTableStatement
          tableFileFormat?
          tableLocation?
          tablePropertiesPrefixed?
-         (KW_AS selectStatement)?
+         (KW_AS selectStatement[true])?
       )
     -> ^(TOK_CREATETABLE $name $ext? ifNotExists?
          ^(TOK_LIKETABLE $likeName?)
@@ -891,8 +900,16 @@ alterTableStatementSuffix
     | alterTblPartitionStatement
     | alterStatementSuffixSkewedby
     | alterStatementSuffixExchangePartition
+    | alterStatementPartitionKeyType
     ;
 
+alterStatementPartitionKeyType
+@init {msgs.push("alter partition key type"); }
+@after {msgs.pop();}
+	: identifier KW_PARTITION KW_COLUMN LPAREN columnNameType RPAREN
+	-> ^(TOK_ALTERTABLE_PARTCOLTYPE identifier columnNameType)
+	;
+
 alterViewStatementSuffix
 @init { msgs.push("alter view statement"); }
 @after { msgs.pop(); }
@@ -903,7 +920,7 @@ alterViewStatementSuffix
         -> ^(TOK_ALTERVIEW_ADDPARTS alterStatementSuffixAddPartitions)
     | alterStatementSuffixDropPartitions
         -> ^(TOK_ALTERVIEW_DROPPARTS alterStatementSuffixDropPartitions)
-    | name=tableName KW_AS selectStatement
+    | name=tableName KW_AS selectStatement[true]
         -> ^(TOK_ALTERVIEW_AS $name selectStatement)
     ;
 
@@ -1049,8 +1066,6 @@ alterTblPartitionStatement
 @after {msgs.pop();}
   : tablePartitionPrefix alterTblPartitionStatementSuffix
   -> ^(TOK_ALTERTABLE_PARTITION tablePartitionPrefix alterTblPartitionStatementSuffix)
-  |Identifier KW_PARTITION KW_COLUMN LPAREN columnNameType RPAREN
-  -> ^(TOK_ALTERTABLE_ALTERPARTS Identifier columnNameType)
   ;
 
 alterTblPartitionStatementSuffix
@@ -1142,7 +1157,7 @@ alterStatementSuffixProtectMode
 @init { msgs.push("alter partition protect mode statement"); }
 @after { msgs.pop(); }
     : alterProtectMode
-    -> ^(TOK_ALTERTABLE_ALTERPARTS_PROTECTMODE alterProtectMode)
+    -> ^(TOK_ALTERTABLE_PROTECTMODE alterProtectMode)
     ;
 
 alterStatementSuffixRenamePart
@@ -1156,7 +1171,7 @@ alterStatementSuffixMergeFiles
 @init { msgs.push(""); }
 @after { msgs.pop(); }
     : KW_CONCATENATE
-    -> ^(TOK_ALTERTABLE_ALTERPARTS_MERGEFILES)
+    -> ^(TOK_ALTERTABLE_MERGEFILES)
     ;
 
 alterProtectMode
@@ -1334,6 +1349,13 @@ showRoleGrants
     -> ^(TOK_SHOW_ROLE_GRANT principalName)
     ;
 
+showRoles
+@init {msgs.push("show roles");}
+@after {msgs.pop();}
+    : KW_SHOW KW_ROLES
+    -> ^(TOK_SHOW_ROLES)
+    ;
+
 showGrants
 @init {msgs.push("show grants");}
 @after {msgs.pop();}
@@ -1361,6 +1383,7 @@ privObjectType
 @init {msgs.push("privilege object type type");}
 @after {msgs.pop();}
     : KW_DATABASE -> ^(TOK_DB_TYPE)
+    | KW_VIEW -> ^(TOK_TABLE_TYPE)
     | KW_TABLE? -> ^(TOK_TABLE_TYPE)
     ;
 
@@ -1466,7 +1489,7 @@ createViewStatement
         (LPAREN columnNameCommentList RPAREN)? tableComment? viewPartition?
         tablePropertiesPrefixed?
         KW_AS
-        selectStatement
+        selectStatement[true]
     -> ^(TOK_CREATEVIEW $name orReplace?
          ifNotExists?
          columnNameCommentList?
@@ -1857,45 +1880,75 @@ unionType
     : KW_UNIONTYPE LESSTHAN colTypeList GREATERTHAN -> ^(TOK_UNIONTYPE colTypeList)
     ;
 
-queryOperator
-@init { msgs.push("query operator"); }
+setOperator
+@init { msgs.push("set operator"); }
 @after { msgs.pop(); }
     : KW_UNION KW_ALL -> ^(TOK_UNION)
     ;
 
-// select statement select ... from ... where ... group by ... order by ...
-queryStatementExpression
-    : queryStatement (queryOperator^ queryStatement)*
+queryStatementExpression[boolean topLevel]
+    :
+    fromStatement[topLevel]
+    | regularBody[topLevel]
     ;
+    
+fromStatement[boolean topLevel]
+: (singleFromStatement  -> singleFromStatement)
+	(u=setOperator r=singleFromStatement
+	  -> ^($u {$fromStatement.tree} $r)
+	)*
+	 -> {u != null && topLevel}? ^(TOK_QUERY
+	       ^(TOK_FROM
+	         ^(TOK_SUBQUERY
+	           {$fromStatement.tree}
+	            {adaptor.create(Identifier, generateUnionAlias())}
+	           )
+	        )
+	       ^(TOK_INSERT 
+	          ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
+	          ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF))
+	        )
+	      )
+    -> {$fromStatement.tree}
+	;
 
-queryStatement
+
+singleFromStatement
     :
     fromClause
     ( b+=body )+ -> ^(TOK_QUERY fromClause body+)
-    | regular_body
     ;
 
-regular_body
+regularBody[boolean topLevel]
    :
-   insertClause
-   selectClause
-   fromClause
-   whereClause?
-   groupByClause?
-   havingClause?
-   orderByClause?
-   clusterByClause?
-   distributeByClause?
-   sortByClause?
-   window_clause?
-   limitClause? -> ^(TOK_QUERY fromClause ^(TOK_INSERT insertClause
-                     selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause?
-                     distributeByClause? sortByClause? window_clause? limitClause?))
+   i=insertClause
+   s=selectStatement[topLevel]
+     {$s.tree.getChild(1).replaceChildren(0, 0, $i.tree);} -> {$s.tree}
    |
-   selectStatement
+   selectStatement[topLevel]
    ;
 
-selectStatement
+ selectStatement[boolean topLevel]
+ : (singleSelectStatement -> singleSelectStatement)
+   (u=setOperator b=singleSelectStatement
+       -> ^($u {$selectStatement.tree} $b)
+   )*
+   -> {u != null && topLevel}? ^(TOK_QUERY
+         ^(TOK_FROM
+           ^(TOK_SUBQUERY
+             {$selectStatement.tree}
+              {adaptor.create(Identifier, generateUnionAlias())}
+             )
+          )
+         ^(TOK_INSERT 
+            ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
+            ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF))
+          )
+        )
+    -> {$selectStatement.tree}
+ ;
+
+singleSelectStatement
    :
    selectClause
    fromClause
@@ -1912,7 +1965,6 @@ selectStatement
                      distributeByClause? sortByClause? window_clause? limitClause?))
    ;
 
-
 body
    :
    insertClause

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Tue Jan 28 05:48:03 2014
@@ -373,7 +373,7 @@ precedenceEqualOperator
 
 subQueryExpression 
     : 
-    LPAREN! selectStatement RPAREN!     
+    LPAREN! selectStatement[true] RPAREN!     
  ;
  
 precedenceEqualExpression
@@ -535,5 +535,5 @@ identifier
 
 nonReserved
     :
-    KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | 
 KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW
 _WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_INNER | KW_DEFINED | KW_ADMIN
+    KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | 
 KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW
 _WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN
     ;

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -129,9 +129,11 @@ public class ImportSemanticAnalyzer exte
         }
         List<Partition> partitions = rv.getValue();
         for (Partition partition : partitions) {
-          AddPartitionDesc partDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(),
+          // TODO: this should not create AddPartitionDesc per partition
+          AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(),
               EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()),
               partition.getSd().getLocation(), partition.getParameters());
+          AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0);
           partDesc.setInputFormat(partition.getSd().getInputFormat());
           partDesc.setOutputFormat(partition.getSd().getOutputFormat());
           partDesc.setNumBuckets(partition.getSd().getNumBuckets());
@@ -142,7 +144,7 @@ public class ImportSemanticAnalyzer exte
           partDesc.setSortCols(partition.getSd().getSortCols());
           partDesc.setLocation(new Path(fromPath,
               Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString());
-          partitionDescs.add(partDesc);
+          partitionDescs.add(partsDesc);
         }
       } catch (IOException e) {
         throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
@@ -188,7 +190,7 @@ public class ImportSemanticAnalyzer exte
             for (Iterator<AddPartitionDesc> partnIter = partitionDescs
                   .listIterator(); partnIter.hasNext();) {
               AddPartitionDesc addPartitionDesc = partnIter.next();
-              if (!found && addPartitionDesc.getPartSpec().equals(partSpec)) {
+              if (!found && addPartitionDesc.getPartition(0).getPartSpec().equals(partSpec)) {
                 found = true;
               } else {
                 partnIter.remove();
@@ -221,12 +223,12 @@ public class ImportSemanticAnalyzer exte
         if (table.isPartitioned()) {
           LOG.debug("table partitioned");
           for (AddPartitionDesc addPartitionDesc : partitionDescs) {
-            if (db.getPartition(table, addPartitionDesc.getPartSpec(), false) == null) {
+            Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
+            if (db.getPartition(table, partSpec, false) == null) {
               rootTasks.add(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc));
             } else {
               throw new SemanticException(
-                  ErrorMsg.PARTITION_EXISTS
-                      .getMsg(partSpecToString(addPartitionDesc.getPartSpec())));
+                  ErrorMsg.PARTITION_EXISTS.getMsg(partSpecToString(partSpec)));
             }
           }
         } else {
@@ -281,7 +283,7 @@ public class ImportSemanticAnalyzer exte
 
   private Task<?> loadTable(URI fromURI, Table table) {
     Path dataPath = new Path(fromURI.toString(), "data");
-    Path tmpPath = new Path(ctx.getExternalTmpFileURI(fromURI));
+    Path tmpPath = ctx.getExternalTmpPath(fromURI);
     Task<?> copyTask = TaskFactory.get(new CopyWork(dataPath,
        tmpPath, false), conf);
     LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
@@ -297,42 +299,43 @@ public class ImportSemanticAnalyzer exte
   private Task<?> addSinglePartition(URI fromURI, FileSystem fs, CreateTableDesc tblDesc,
       Table table, Warehouse wh,
       AddPartitionDesc addPartitionDesc) throws MetaException, IOException, HiveException {
+    AddPartitionDesc.OnePartitionDesc partSpec = addPartitionDesc.getPartition(0);
     if (tblDesc.isExternal() && tblDesc.getLocation() == null) {
       LOG.debug("Importing in-place: adding AddPart for partition "
-          + partSpecToString(addPartitionDesc.getPartSpec()));
+          + partSpecToString(partSpec.getPartSpec()));
       // addPartitionDesc already has the right partition location
       Task<?> addPartTask = TaskFactory.get(new DDLWork(getInputs(),
           getOutputs(), addPartitionDesc), conf);
       return addPartTask;
     } else {
-      String srcLocation = addPartitionDesc.getLocation();
+      String srcLocation = partSpec.getLocation();
       Path tgtPath = null;
       if (tblDesc.getLocation() == null) {
         if (table.getDataLocation() != null) {
           tgtPath = new Path(table.getDataLocation().toString(),
-              Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
+              Warehouse.makePartPath(partSpec.getPartSpec()));
         } else {
           tgtPath = new Path(wh.getTablePath(
               db.getDatabaseCurrent(), tblDesc.getTableName()),
-              Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
+              Warehouse.makePartPath(partSpec.getPartSpec()));
         }
       } else {
         tgtPath = new Path(tblDesc.getLocation(),
-            Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
+            Warehouse.makePartPath(partSpec.getPartSpec()));
       }
       checkTargetLocationEmpty(fs, tgtPath);
-      addPartitionDesc.setLocation(tgtPath.toString());
+      partSpec.setLocation(tgtPath.toString());
       LOG.debug("adding dependent CopyWork/AddPart/MoveWork for partition "
-          + partSpecToString(addPartitionDesc.getPartSpec())
+          + partSpecToString(partSpec.getPartSpec())
           + " with source location: " + srcLocation);
-      Path tmpPath = new Path(ctx.getExternalTmpFileURI(fromURI));
+      Path tmpPath = ctx.getExternalTmpPath(fromURI);
       Task<?> copyTask = TaskFactory.get(new CopyWork(new Path(srcLocation),
           tmpPath, false), conf);
       Task<?> addPartTask = TaskFactory.get(new DDLWork(getInputs(),
           getOutputs(), addPartitionDesc), conf);
       LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
           Utilities.getTableDesc(table),
-          addPartitionDesc.getPartSpec(), true);
+          partSpec.getPartSpec(), true);
       loadTableWork.setInheritTableSpecs(false);
       Task<?> loadPartTask = TaskFactory.get(new MoveWork(
           getInputs(), getOutputs(), loadTableWork, null, false),
@@ -399,7 +402,7 @@ public class ImportSemanticAnalyzer exte
         if (tableDesc.getLocation() != null) { // IMPORT statement specified
                                                // location
           if (!table.getDataLocation()
-              .equals(new URI(tableDesc.getLocation()))) {
+              .equals(new Path(tableDesc.getLocation()))) {
             throw new SemanticException(
                 ErrorMsg.INCOMPATIBLE_SCHEMA.getMsg(" Location does not match"));
           }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -206,8 +206,8 @@ public class LoadSemanticAnalyzer extend
       throw new SemanticException(ErrorMsg.LOAD_INTO_STORED_AS_DIR.getMsg());
     }
 
-    URI toURI = (ts.partHandle != null) ? ts.partHandle.getDataLocation()
-        : ts.tableHandle.getDataLocation();
+    URI toURI = ((ts.partHandle != null) ? ts.partHandle.getDataLocation()
+        : ts.tableHandle.getDataLocation()).toUri();
 
     List<FieldSchema> parts = ts.tableHandle.getPartitionKeys();
     if ((parts != null && parts.size() > 0)
@@ -226,7 +226,7 @@ public class LoadSemanticAnalyzer extend
       // might seem redundant in the case
       // that the hive warehouse is also located in the local file system - but
       // that's just a test case.
-      String copyURIStr = ctx.getExternalTmpFileURI(toURI);
+      String copyURIStr = ctx.getExternalTmpPath(toURI).toString();
       URI copyURI = URI.create(copyURIStr);
       rTask = TaskFactory.get(new CopyWork(new Path(fromURI), new Path(copyURI)), conf);
       fromURI = copyURI;
@@ -234,7 +234,6 @@ public class LoadSemanticAnalyzer extend
 
     // create final load/move work
 
-    String loadTmpPath = ctx.getExternalTmpFileURI(toURI);
     Map<String, String> partSpec = ts.getPartSpec();
     if (partSpec == null) {
       partSpec = new LinkedHashMap<String, String>();

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java Tue Jan 28 05:48:03 2014
@@ -345,6 +345,9 @@ public class PTFTranslator {
   private WindowFunctionDef translate(WindowTableFunctionDef wdwTFnDef,
       WindowFunctionSpec spec) throws SemanticException {
     WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(spec.getName());
+    if (wFnInfo == null) {
+      throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(spec.getName()));
+    }
     WindowFunctionDef def = new WindowFunctionDef();
     def.setName(spec.getName());
     def.setAlias(spec.getAlias());

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1561947&r1=1561946&r2=1561947&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Jan 28 05:48:03 2014
@@ -181,7 +181,9 @@ import org.apache.hadoop.hive.serde2.typ
 import org.apache.hadoop.mapred.InputFormat;
 
 /**
- * Implementation of the semantic analyzer.
+ * Implementation of the semantic analyzer. It generates the query plan.
+ * There are other specific semantic analyzers for some hive operations such as
+ * DDLSemanticAnalyzer for ddl operations.
  */
 
 public class SemanticAnalyzer extends BaseSemanticAnalyzer {
@@ -967,10 +969,9 @@ public class SemanticAnalyzer extends Ba
         break;
 
       case HiveParser.TOK_UNION:
-        // currently, we dont support subq1 union subq2 - the user has to
-        // explicitly say:
-        // select * from (subq1 union subq2) subqalias
         if (!qbp.getIsSubQ()) {
+          // this shouldn't happen. The parser should have converted the union to be
+          // contained in a subquery. Just in case, we keep the error as a fallback.
           throw new SemanticException(generateErrorMessage(ast,
               ErrorMsg.UNION_NOTIN_SUBQ.getMsg()));
         }
@@ -1021,7 +1022,7 @@ public class SemanticAnalyzer extends Ba
           } catch (HiveException e) {
             LOG.info("Error while getting metadata : ", e);
           }
-          validatePartSpec(table, partition, (ASTNode)tab, conf);
+          validatePartSpec(table, partition, (ASTNode)tab, conf, false);
         }
         skipRecursion = false;
         break;
@@ -1263,8 +1264,8 @@ public class SemanticAnalyzer extends Ba
                 throw new SemanticException(e);
               }
               try {
-                fname = ctx.getExternalTmpFileURI(
-                    FileUtils.makeQualified(location, conf).toUri());
+                fname = ctx.getExternalTmpPath(
+                    FileUtils.makeQualified(location, conf).toUri()).toString();
               } catch (Exception e) {
                 throw new SemanticException(generateErrorMessage(ast,
                     "Error creating temporary folder on: " + location.toString()), e);
@@ -1278,7 +1279,7 @@ public class SemanticAnalyzer extends Ba
               }
             } else {
               qb.setIsQuery(true);
-              fname = ctx.getMRTmpFileURI();
+              fname = ctx.getMRTmpPath().toString();
               ctx.setResDir(new Path(fname));
             }
           }
@@ -5264,7 +5265,7 @@ public class SemanticAnalyzer extends Ba
 
     Table dest_tab = null; // destination table if any
     Partition dest_part = null;// destination partition if any
-    String queryTmpdir = null; // the intermediate destination directory
+    Path queryTmpdir = null; // the intermediate destination directory
     Path dest_path = null; // the final destination directory
     TableDesc table_desc = null;
     int currentTableId = 0;
@@ -5307,7 +5308,7 @@ public class SemanticAnalyzer extends Ba
         }
         dpCtx = qbm.getDPCtx(dest);
         if (dpCtx == null) {
-          Utilities.validatePartSpecColumnNames(dest_tab, partSpec);
+          dest_tab.validatePartColumnNames(partSpec, false);
           dpCtx = new DynamicPartitionCtx(dest_tab, partSpec,
               conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME),
               conf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTSPERNODE));
@@ -5330,9 +5331,9 @@ public class SemanticAnalyzer extends Ba
 
       boolean isNonNativeTable = dest_tab.isNonNative();
       if (isNonNativeTable) {
-        queryTmpdir = dest_path.toUri().getPath();
+        queryTmpdir = dest_path;
       } else {
-        queryTmpdir = ctx.getExternalTmpFileURI(dest_path.toUri());
+        queryTmpdir = ctx.getExternalTmpPath(dest_path.toUri());
       }
       if (dpCtx != null) {
         // set the root of the temporay path where dynamic partition columns will populate
@@ -5355,7 +5356,7 @@ public class SemanticAnalyzer extends Ba
       // Create the work for moving the table
       // NOTE: specify Dynamic partitions in dest_tab for WriteEntity
       if (!isNonNativeTable) {
-        ltd = new LoadTableDesc(new Path(queryTmpdir),table_desc, dpCtx);
+        ltd = new LoadTableDesc(queryTmpdir,table_desc, dpCtx);
         ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
             dest_tab.getTableName()));
         ltd.setLbCtx(lbCtx);
@@ -5418,14 +5419,14 @@ public class SemanticAnalyzer extends Ba
       }
 
       Path tabPath = dest_tab.getPath();
-      Path partPath = dest_part.getPartitionPath();
+      Path partPath = dest_part.getDataLocation();
 
       // if the table is in a different dfs than the partition,
       // replace the partition's dfs with the table's dfs.
       dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri()
           .getAuthority(), partPath.toUri().getPath());
 
-      queryTmpdir = ctx.getExternalTmpFileURI(dest_path.toUri());
+      queryTmpdir = ctx.getExternalTmpPath(dest_path.toUri());
       table_desc = Utilities.getTableDesc(dest_tab);
 
       // Add sorting/bucketing if needed
@@ -5438,7 +5439,7 @@ public class SemanticAnalyzer extends Ba
       lbCtx = constructListBucketingCtx(dest_part.getSkewedColNames(),
           dest_part.getSkewedColValues(), dest_part.getSkewedColValueLocationMaps(),
           dest_part.isStoredAsSubDirectories(), conf);
-      ltd = new LoadTableDesc(new Path(queryTmpdir), table_desc, dest_part.getSpec());
+      ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec());
       ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
           dest_tab.getTableName()));
       ltd.setLbCtx(lbCtx);
@@ -5469,18 +5470,18 @@ public class SemanticAnalyzer extends Ba
       // fall through
     case QBMetaData.DEST_DFS_FILE: {
       dest_path = new Path(qbm.getDestFileForAlias(dest));
-      
+
       if (isLocal) {
         // for local directory - we always write to map-red intermediate
         // store and then copy to local fs
-        queryTmpdir = ctx.getMRTmpFileURI();
+        queryTmpdir = ctx.getMRTmpPath();
       } else {
         // otherwise write to the file system implied by the directory
         // no copy is required. we may want to revisit this policy in future
 
         try {
           Path qPath = FileUtils.makeQualified(dest_path, conf);
-          queryTmpdir = ctx.getExternalTmpFileURI(qPath.toUri());
+          queryTmpdir = ctx.getExternalTmpPath(qPath.toUri());
         } catch (Exception e) {
           throw new SemanticException("Error creating temporary folder on: "
               + dest_path, e);
@@ -5558,7 +5559,7 @@ public class SemanticAnalyzer extends Ba
       }
 
       boolean isDfsDir = (dest_type.intValue() == QBMetaData.DEST_DFS_FILE);
-      loadFileWork.add(new LoadFileDesc(tblDesc, new Path(queryTmpdir), dest_path, isDfsDir, cols,
+      loadFileWork.add(new LoadFileDesc(tblDesc, queryTmpdir, dest_path, isDfsDir, cols,
           colTypes));
 
       if (tblDesc == null) {
@@ -5638,7 +5639,7 @@ public class SemanticAnalyzer extends Ba
     // the same as directory name. The directory name
     // can be changed in the optimizer but the key should not be changed
     // it should be the same as the MoveWork's sourceDir.
-    fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName());
+    fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName().toString());
 
     if (dest_part != null) {
       try {



Mime
View raw message