hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1501145 [4/13] - in /hive/branches/vectorization: ./ beeline/src/java/org/apache/hive/beeline/ beeline/src/test/org/apache/hive/beeline/src/test/ cli/src/java/org/apache/hadoop/hive/cli/ common/src/java/org/apache/hadoop/hive/common/metric...
Date Tue, 09 Jul 2013 09:07:43 GMT
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java Tue Jul  9 09:07:35 2013
@@ -142,6 +142,17 @@ public class ExprNodeGenericFuncEvaluato
   }
 
   @Override
+  public boolean isStateful() {
+    boolean result = FunctionRegistry.isStateful(genericUDF);
+    for (ExprNodeEvaluator child : children) {
+      if(result = result || child.isStateful()) {
+        return result;
+      }
+    }
+    return result;
+  }
+
+  @Override
   protected Object _evaluate(Object row, int version) throws HiveException {
     rowObject = row;
     if (ObjectInspectorUtils.isConstantObjectInspector(outputOI) &&

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Tue Jul  9 09:07:35 2013
@@ -493,11 +493,15 @@ public class FetchOperator implements Se
   public boolean pushRow() throws IOException, HiveException {
     InspectableObject row = getNextRow();
     if (row != null) {
-      operator.process(row.o, 0);
+      pushRow(row);
     }
     return row != null;
   }
 
+  protected void pushRow(InspectableObject row) throws HiveException {
+    operator.process(row.o, 0);
+  }
+
   private transient final InspectableObject inspectable = new InspectableObject();
 
   /**

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Tue Jul  9 09:07:35 2013
@@ -163,19 +163,6 @@ public class FetchTask extends Task<Fetc
     return "FETCH";
   }
 
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    String s = work.getTblDir();
-    if ((s != null) && ctx.isMRTmpFileURI(s)) {
-      work.setTblDir(ctx.localizeMRTmpFileURI(s));
-    }
-
-    ArrayList<String> ls = work.getPartDir();
-    if (ls != null) {
-      ctx.localizePaths(ls);
-    }
-  }
-
   /**
    * Clear the Fetch Operator.
    *

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue Jul  9 09:07:35 2013
@@ -1105,6 +1105,10 @@ public final class FunctionRegistry {
       GenericUDFBridge bridge = (GenericUDFBridge) genericUDF;
       return new GenericUDFBridge(bridge.getUdfName(), bridge.isOperator(),
           bridge.getUdfClass());
+    } else if (genericUDF instanceof GenericUDFMacro) {
+      GenericUDFMacro bridge = (GenericUDFMacro) genericUDF;
+      return new GenericUDFMacro(bridge.getMacroName(), bridge.getBody(),
+          bridge.getColNames(), bridge.getColTypes());
     }
 
     return (GenericUDF) ReflectionUtils
@@ -1173,6 +1177,11 @@ public final class FunctionRegistry {
       }
     }
 
+    if (genericUDF instanceof GenericUDFMacro) {
+      GenericUDFMacro macro = (GenericUDFMacro) (genericUDF);
+      return macro.isDeterministic();
+    }
+
     return true;
   }
 
@@ -1193,6 +1202,11 @@ public final class FunctionRegistry {
       }
     }
 
+    if (genericUDF instanceof GenericUDFMacro) {
+      GenericUDFMacro macro = (GenericUDFMacro) (genericUDF);
+      return macro.isStateful();
+    }
+
     return false;
   }
 
@@ -1303,6 +1317,27 @@ public final class FunctionRegistry {
   }
 
   /**
+   * Registers thae appropriate kind of temporary function based on a class's
+   * type.
+   *
+   * @param macroName name under which to register the macro
+   *
+   * @param body the expression which the macro evaluates to
+   *
+   * @param colNames the names of the arguments to the macro
+   *
+   * @param colTypes the types of the arguments to the macro
+   */
+  public static void registerTemporaryMacro(
+    String macroName, ExprNodeDesc body,
+    List<String> colNames, List<TypeInfo> colTypes) {
+
+    FunctionInfo fI = new FunctionInfo(false, macroName,
+        new GenericUDFMacro(macroName, body, colNames, colTypes));
+    mFunctions.put(macroName.toLowerCase(), fI);
+  }
+
+  /**
    * Registers Hive functions from a plugin jar, using metadata from
    * the jar's META-INF/class-info.xml.
    *

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Tue Jul  9 09:07:35 2013
@@ -28,6 +28,8 @@ import org.apache.hadoop.hive.ql.QueryPl
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.CreateFunctionDesc;
 import org.apache.hadoop.hive.ql.plan.DropFunctionDesc;
+import org.apache.hadoop.hive.ql.plan.CreateMacroDesc;
+import org.apache.hadoop.hive.ql.plan.DropMacroDesc;
 import org.apache.hadoop.hive.ql.plan.FunctionWork;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
@@ -67,6 +69,16 @@ public class FunctionTask extends Task<F
     if (dropFunctionDesc != null) {
       return dropFunction(dropFunctionDesc);
     }
+
+    CreateMacroDesc createMacroDesc = work.getCreateMacroDesc();
+    if (createMacroDesc != null) {
+      return createMacro(createMacroDesc);
+    }
+
+    DropMacroDesc dropMacroDesc = work.getDropMacroDesc();
+    if (dropMacroDesc != null) {
+      return dropMacro(dropMacroDesc);
+    }
     return 0;
   }
 
@@ -89,6 +101,26 @@ public class FunctionTask extends Task<F
     }
   }
 
+  private int createMacro(CreateMacroDesc createMacroDesc) {
+    FunctionRegistry.registerTemporaryMacro(
+      createMacroDesc.getMacroName(),
+      createMacroDesc.getBody(),
+      createMacroDesc.getColNames(),
+      createMacroDesc.getColTypes());
+    return 0;
+  }
+
+  private int dropMacro(DropMacroDesc dropMacroDesc) {
+    try {
+      FunctionRegistry.unregisterTemporaryUDF(dropMacroDesc
+          .getMacroName());
+      return 0;
+    } catch (HiveException e) {
+      LOG.info("drop macro: " + StringUtils.stringifyException(e));
+      return 1;
+    }
+  }
+
   private int dropFunction(DropFunctionDesc dropFunctionDesc) {
     try {
       FunctionRegistry.unregisterTemporaryUDF(dropFunctionDesc
@@ -114,9 +146,4 @@ public class FunctionTask extends Task<F
   public String getName() {
     return "FUNCTION";
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    throw new RuntimeException ("Unexpected call");
-  }
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Tue Jul  9 09:07:35 2013
@@ -93,8 +93,9 @@ public class JoinOperator extends Common
           .toString());
       List keyObject = (List) soi.getStructFieldData(row, sf);
       // Are we consuming too much memory
-      if (alias == numAliases - 1 && !(handleSkewJoin && skewJoinKeyContext.currBigKeyTag >= 0)) {
-        if (sz == joinEmitInterval) {
+      if (alias == numAliases - 1 && !(handleSkewJoin && skewJoinKeyContext.currBigKeyTag >= 0) &&
+          !hasLeftSemiJoin) {
+        if (sz == joinEmitInterval && !hasFilter(alias)) {
           // The input is sorted by alias, so if we are already in the last join
           // operand,
           // we can emit some results now.

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Tue Jul  9 09:07:35 2013
@@ -242,10 +242,12 @@ public class MapOperator extends Operato
       SerDeException {
     PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
     LinkedHashMap<String, String> partSpec = pd.getPartSpec();
-    // Use tblProps in case of unpartitioned tables
+    // Use table properties in case of unpartitioned tables,
+    // and the union of table properties and partition properties, with partition
+    // taking precedence
     Properties partProps =
         (pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
-            pd.getTableDesc().getProperties() : pd.getProperties();
+            pd.getTableDesc().getProperties() : pd.getOverlayedProperties();
 
     Class serdeclass = pd.getDeserializerClass();
     if (serdeclass == null) {
@@ -409,7 +411,7 @@ public class MapOperator extends Operato
         // If the partition does not exist, use table properties
         Properties partProps =
             (pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
-                tblProps : pd.getProperties();
+                tblProps : pd.getOverlayedProperties();
 
         Class sdclass = pd.getDeserializerClass();
         if (sdclass == null) {

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java Tue Jul  9 09:07:35 2013
@@ -448,11 +448,6 @@ public class MapredLocalTask extends Tas
   }
 
   @Override
-  public void localizeMRTmpFilesImpl(Context ctx) {
-
-  }
-
-  @Override
   public boolean isMapRedLocalTask() {
     return true;
   }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Tue Jul  9 09:07:35 2013
@@ -537,10 +537,4 @@ public class MoveTask extends Task<MoveW
   public String getName() {
     return "MOVE";
   }
-
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no-op
-  }
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java Tue Jul  9 09:07:35 2013
@@ -1,4 +1,9 @@
 /**
+ <<<<<<< HEAD
+ =======
+ * Copyright 2010 The Apache Software Foundation
+ *
+ >>>>>>> HIVE-1402 [jira] Add parallel ORDER BY to Hive
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -28,6 +33,11 @@ public class OperatorUtils {
     return findOperator(start, clazz, new HashSet<T>());
   }
 
+  public static <T> T findSingleOperator(Operator<?> start, Class<T> clazz) {
+    Set<T> found = findOperator(start, clazz, new HashSet<T>());
+    return found.size() == 1 ? found.iterator().next() : null;
+  }
+
   public static <T> Set<T> findOperators(Collection<Operator<?>> starts, Class<T> clazz) {
     Set<T> found = new HashSet<T>();
     for (Operator<?> start : starts) {

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java Tue Jul  9 09:07:35 2013
@@ -107,6 +107,7 @@ public class PTFOperator extends Operato
         processInputPartition();
       }
     }
+    inputPart.close();
 	}
 
 	@Override

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPartition.java Tue Jul  9 09:07:35 2013
@@ -152,6 +152,10 @@ public class PTFPartition
     return new PItr(start, end);
   }
 
+  public void close() {
+    elems.close();
+  }
+
   class PItr implements PTFPartitionIterator<Object>
   {
     int idx;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFPersistence.java Tue Jul  9 09:07:35 2013
@@ -37,6 +37,8 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.ConcurrentModificationException;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
@@ -287,6 +289,11 @@ public class PTFPersistence {
       bldr.append("]\n");
     }
 
+    public void close() {
+      bytes = null;
+      offsetsArray = null;
+    }
+
     class WIterator implements Iterator<Writable>
     {
       Writable wObj;
@@ -403,6 +410,12 @@ public class PTFPersistence {
 
   public static class PartitionedByteBasedList extends ByteBasedList
   {
+    private static final ShutdownHook hook = new ShutdownHook();
+
+    static {
+      Runtime.getRuntime().addShutdownHook(hook);
+    }
+
     ArrayList<ByteBasedList> partitions;
     ArrayList<Integer> partitionOffsets;
     ArrayList<File> reusableFiles;
@@ -413,8 +426,7 @@ public class PTFPersistence {
     {
       this.batchSize = batchSize;
       currentSize = 0;
-      dir = PartitionedByteBasedList.createTempDir();
-      Runtime.getRuntime().addShutdownHook(new ShutdownHook(dir));
+      hook.register(dir = PartitionedByteBasedList.createTempDir());
 
       partitions = new ArrayList<ByteBasedList>();
       partitionOffsets = new ArrayList<Integer>();
@@ -446,6 +458,22 @@ public class PTFPersistence {
       }
     }
 
+    @Override
+    public void close() {
+      super.close();
+      reusableFiles.clear();
+      partitionOffsets.clear();
+      for (ByteBasedList partition : partitions) {
+        partition.close();
+      }
+      partitions.clear();
+      try {
+        PartitionedByteBasedList.deleteRecursively(dir);
+      } catch (Exception e) {
+      }
+      hook.unregister(dir);
+    }
+
     private void addPartition() throws HiveException
     {
       try
@@ -636,11 +664,14 @@ public class PTFPersistence {
 
     static class ShutdownHook extends Thread
     {
-      File dir;
+      private final Set<File> dirs = new LinkedHashSet<File>();
 
-      public ShutdownHook(File dir)
-      {
-        this.dir = dir;
+      public void register(File dir) {
+        dirs.add(dir);
+      }
+
+      public void unregister(File dir) {
+        dirs.remove(dir);
       }
 
       @Override
@@ -648,9 +679,11 @@ public class PTFPersistence {
       {
         try
         {
-          PartitionedByteBasedList.deleteRecursively(dir);
+          for (File dir : dirs) {
+            PartitionedByteBasedList.deleteRecursively(dir);
+          }
         }
-        catch(IOException ie)
+        catch(Exception ie)
         {
         }
       }
@@ -852,6 +885,21 @@ public class PTFPersistence {
       throw new HiveException("Reset on PersistentByteBasedList not supported");
     }
 
+    @Override
+    public void close() {
+      super.close();
+      ByteBasedList list = memList.get();
+      if (list != null) {
+        list.close();
+      }
+      memList.clear();
+      try {
+        PartitionedByteBasedList.deleteRecursively(file);
+      } catch (Exception e) {
+        // ignore
+      }
+    }
+
     private ByteBasedList getList() throws HiveException
     {
       PTFPersistence.lock(lock.readLock());

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Tue Jul  9 09:07:35 2013
@@ -263,11 +263,6 @@ public class StatsTask extends Task<Stat
     return "STATS";
   }
 
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // Nothing to do for StatsTask here.
-  }
-
   private int aggregateStats() {
 
     StatsAggregator statsAggregator = null;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Tue Jul  9 09:07:35 2013
@@ -95,6 +95,12 @@ public abstract class Task<T extends Ser
 
   protected List<Task<? extends Serializable>> childTasks;
   protected List<Task<? extends Serializable>> parentTasks;
+  /**
+   * this can be set by the Task, to provide more info about the failure in TaskResult
+   * where the Driver can find it.  This is checked if {@link Task#execute(org.apache.hadoop.hive.ql.DriverContext)}
+   * returns non-0 code.
+   */
+  private Throwable exception;
 
   public Task() {
     isdone = false;
@@ -384,37 +390,6 @@ public abstract class Task<T extends Ser
   public abstract StageType getType();
 
   /**
-   * If this task uses any map-reduce intermediate data (either for reading or for writing),
-   * localize them (using the supplied Context). Map-Reduce intermediate directories are allocated
-   * using Context.getMRTmpFileURI() and can be localized using localizeMRTmpFileURI().
-   *
-   * This method is declared abstract to force any task code to explicitly deal with this aspect of
-   * execution.
-   *
-   * @param ctx
-   *          context object with which to localize
-   */
-  abstract protected void localizeMRTmpFilesImpl(Context ctx);
-
-  /**
-   * Localize a task tree
-   *
-   * @param ctx
-   *          context object with which to localize
-   */
-  public final void localizeMRTmpFiles(Context ctx) {
-    localizeMRTmpFilesImpl(ctx);
-
-    if (childTasks == null) {
-      return;
-    }
-
-    for (Task<? extends Serializable> t : childTasks) {
-      t.localizeMRTmpFiles(ctx);
-    }
-  }
-
-  /**
    * Subscribe the feed of publisher. To prevent cycles, a task can only subscribe to its ancestor.
    * Feed is a generic form of execution-time feedback (type, value) pair from one task to another
    * task. Examples include dynamic partitions (which are only available at execution time). The
@@ -525,4 +500,10 @@ public abstract class Task<T extends Ser
   public List<FieldSchema> getResultSchema() {
     return null;
   }
+  Throwable getException() {
+    return exception;
+  }
+  void setException(Throwable ex) {
+    exception = ex;
+  }
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskResult.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskResult.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskResult.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskResult.java Tue Jul  9 09:07:35 2013
@@ -20,11 +20,13 @@ package org.apache.hadoop.hive.ql.exec;
 
 /**
  * TaskResult implementation.
+ * Note that different threads may be reading/writing this object
  **/
 
 public class TaskResult {
-  protected int exitVal;
-  protected boolean runStatus;
+  protected volatile int exitVal;
+  protected volatile boolean runStatus;
+  private volatile Throwable taskError;
 
   public TaskResult() {
     exitVal = -1;
@@ -35,11 +37,21 @@ public class TaskResult {
     this.exitVal = exitVal;
     setRunning(false);
   }
+  public void setExitVal(int exitVal, Throwable taskError) {
+    this.setExitVal(exitVal);
+    this.taskError = taskError;
+  }
 
   public int getExitVal() {
     return exitVal;
   }
 
+  /**
+   * @return may contain details of the error which caused the task to fail or null
+   */
+  public Throwable getTaskError() {
+    return taskError;
+  }
   public boolean isRunning() {
     return runStatus;
   }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java Tue Jul  9 09:07:35 2013
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.ql.exec;
 
 import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.hadoop.hive.ql.session.SessionState;
 
@@ -30,6 +31,13 @@ public class TaskRunner extends Thread {
   protected Task<? extends Serializable> tsk;
   protected TaskResult result;
   protected SessionState ss;
+  private static AtomicLong taskCounter = new AtomicLong(0);
+  private static ThreadLocal<Long> taskRunnerID = new ThreadLocal<Long>() {
+    @Override
+    protected Long initialValue() {
+      return taskCounter.incrementAndGet();
+    }
+  };
 
   public TaskRunner(Task<? extends Serializable> tsk, TaskResult result) {
     this.tsk = tsk;
@@ -58,7 +66,10 @@ public class TaskRunner extends Thread {
     } catch (Throwable t) {
       t.printStackTrace();
     }
-    result.setExitVal(exitVal);
+    result.setExitVal(exitVal, tsk.getException());
   }
 
+  public static long getTaskRunnerID () {
+    return taskRunnerID.get();
+  }
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java Tue Jul  9 09:07:35 2013
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.ql.exec;
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -28,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.UDTFDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
 import org.apache.hadoop.hive.ql.udf.generic.UDTFCollector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
@@ -45,7 +47,10 @@ public class UDTFOperator extends Operat
 
   ObjectInspector[] udtfInputOIs = null;
   Object[] objToSendToUDTF = null;
-  Object[] forwardObj = new Object[1];
+
+  GenericUDTF genericUDTF;
+  UDTFCollector collector;
+  List outerObj;
 
   /**
    * sends periodic reports back to the tracker.
@@ -54,7 +59,10 @@ public class UDTFOperator extends Operat
 
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {
-    conf.getGenericUDTF().setCollector(new UDTFCollector(this));
+    genericUDTF = conf.getGenericUDTF();
+    collector = new UDTFCollector(this);
+
+    genericUDTF.setCollector(collector);
 
     // Make an object inspector [] of the arguments to the UDTF
     List<? extends StructField> inputFields =
@@ -68,10 +76,13 @@ public class UDTFOperator extends Operat
 
     MapredContext context = MapredContext.get();
     if (context != null) {
-      context.setup(conf.getGenericUDTF());
+      context.setup(genericUDTF);
     }
-    StructObjectInspector udtfOutputOI = conf.getGenericUDTF().initialize(
+    StructObjectInspector udtfOutputOI = genericUDTF.initialize(
         udtfInputOIs);
+    if (conf.isOuterLV()) {
+      outerObj = Arrays.asList(new Object[udtfOutputOI.getAllStructFieldRefs().size()]);
+    }
 
     // Since we're passing the object output by the UDTF directly to the next
     // operator, we can use the same OI.
@@ -100,8 +111,11 @@ public class UDTFOperator extends Operat
       objToSendToUDTF[i] = soi.getStructFieldData(row, fields.get(i));
     }
 
-    conf.getGenericUDTF().process(objToSendToUDTF);
-
+    genericUDTF.process(objToSendToUDTF);
+    if (conf.isOuterLV() && collector.getCounter() == 0) {
+      collector.collect(outerObj);
+    }
+    collector.reset();
   }
 
   /**

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Jul  9 09:07:35 2013
@@ -1313,6 +1313,18 @@ public final class Utilities {
   }
 
   /**
+   * returns null if path is not exist
+   */
+  public static FileStatus[] listStatusIfExists(Path path, FileSystem fs) throws IOException {
+    try {
+      return fs.listStatus(path);
+    } catch (FileNotFoundException e) {
+      // FS in hadoop 2.0 throws FNF instead of returning null
+      return null;
+    }
+  }
+
+  /**
    * Get all file status from a root path and recursively go deep into certain levels.
    *
    * @param path

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/DataCorruptErrorHeuristic.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/DataCorruptErrorHeuristic.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/DataCorruptErrorHeuristic.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/DataCorruptErrorHeuristic.java Tue Jul  9 09:07:35 2013
@@ -36,7 +36,7 @@ import java.util.regex.Pattern;
 
 public class DataCorruptErrorHeuristic extends RegexErrorHeuristic {
 
-  private static final String SPLIT_REGEX = "split:.*";
+  private static final String SPLIT_REGEX = "split:\\s*([^\\s]+)";
   private static final String EXCEPTION_REGEX = "EOFException";
 
   public DataCorruptErrorHeuristic() {
@@ -55,14 +55,13 @@ public class DataCorruptErrorHeuristic e
           rll.get(SPLIT_REGEX).size() > 0) {
 
         // There should only be a single split line...
-        assert(rll.get(SPLIT_REGEX).size()==1);
         String splitLogLine = rll.get(SPLIT_REGEX).get(0);
 
         // Extract only 'split: hdfs://...'
         Pattern p = Pattern.compile(SPLIT_REGEX, Pattern.CASE_INSENSITIVE);
         Matcher m = p.matcher(splitLogLine);
         m.find();
-        String splitStr = m.group();
+        String splitStr = m.group(1);
 
         es = new ErrorAndSolution(
             "Data file " + splitStr + " is corrupted.",

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/ErrorAndSolution.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/ErrorAndSolution.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/ErrorAndSolution.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/ErrorAndSolution.java Tue Jul  9 09:07:35 2013
@@ -23,8 +23,8 @@ package org.apache.hadoop.hive.ql.exec.e
  */
 public class ErrorAndSolution {
 
-  private String error = null;
-  private String solution = null;
+  private final String error;
+  private final String solution;
 
   ErrorAndSolution(String error, String solution) {
     this.error = error;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/RegexErrorHeuristic.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/RegexErrorHeuristic.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/RegexErrorHeuristic.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/RegexErrorHeuristic.java Tue Jul  9 09:07:35 2013
@@ -40,7 +40,6 @@ import org.apache.hadoop.mapred.JobConf;
  */
 public abstract class RegexErrorHeuristic implements ErrorHeuristic {
 
-  private String query = null;
   private JobConf conf = null;
 
   // Pattern to look for in the hive query and whether it matched
@@ -86,10 +85,9 @@ public abstract class RegexErrorHeuristi
    * Before init is called, logRegexes and queryRegexes should be populated.
    */
   public void init(String query, JobConf conf) {
-    this.query = query;
     this.conf = conf;
 
-    assert((logRegexes!=null) && (queryRegex != null));
+    assert(queryRegex != null);
 
     Pattern queryPattern = Pattern.compile(queryRegex, Pattern.CASE_INSENSITIVE);
     queryMatches = queryPattern.matcher(query).find();
@@ -98,7 +96,6 @@ public abstract class RegexErrorHeuristi
       regexToPattern.put(regex, Pattern.compile(regex, Pattern.CASE_INSENSITIVE));
       regexToLogLines.put(regex, new ArrayList<String>());
     }
-
   }
 
   @Override

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java Tue Jul  9 09:07:35 2013
@@ -47,13 +47,11 @@ public class TaskLogProcessor {
     new HashMap<ErrorHeuristic, HeuristicStats>();
   private final List<String> taskLogUrls = new ArrayList<String>();
 
-  private JobConf conf = null;
   // Query is the hive query string i.e. "SELECT * FROM src;" associated with
   // this set of tasks logs
-  private String query = null;
+  private final String query;
 
   public TaskLogProcessor(JobConf conf) {
-    this.conf = conf;
     query = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING);
 
     heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats());
@@ -197,7 +195,6 @@ public class TaskLogProcessor {
       try {
         in = new BufferedReader(
             new InputStreamReader(taskAttemptLogUrl.openStream()));
-        String inputLine;
         String lastLine = null;
         boolean lastLineMatched = false;
         List<String> stackTrace = null;
@@ -207,9 +204,20 @@ public class TaskLogProcessor {
         Pattern endStackTracePattern =
             Pattern.compile("^\t... [0-9]+ more.*", Pattern.CASE_INSENSITIVE);
 
-        while ((inputLine =
-          ShimLoader.getHadoopShims().unquoteHtmlChars(in.readLine())) != null) {
-
+        String inputLine;
+        while (true) {
+          inputLine = in.readLine();
+          if (inputLine == null) { // EOF:
+            if (stackTrace != null) {
+              stackTraces.add(stackTrace);
+              stackTrace = null;
+            }
+            break;
+          }
+          
+          inputLine =
+              ShimLoader.getHadoopShims().unquoteHtmlChars(inputLine);
+          
           if (stackTracePattern.matcher(inputLine).matches() ||
               endStackTracePattern.matcher(inputLine).matches()) {
             // We are in a stack trace

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java Tue Jul  9 09:07:35 2013
@@ -287,7 +287,7 @@ public class HiveHistory {
       String key = ent.getKey();
       String val = ent.getValue();
       if(val != null) {
-        val = val.replace('\n', ' ');
+        val = val.replace(System.getProperty("line.separator"), " ");
       }
       sb.append(key + "=\"" + val + "\"");
 

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java Tue Jul  9 09:07:35 2013
@@ -98,9 +98,4 @@ public class IndexMetadataChangeTask ext
   public StageType getType() {
     return StageType.DDL;
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-  }
-
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java Tue Jul  9 09:07:35 2013
@@ -118,6 +118,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final boolean readBoolean() throws IOException {
     int temp = in.read();
     if (temp < 0) {
@@ -135,6 +136,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final byte readByte() throws IOException {
     int temp = in.read();
     if (temp < 0) {
@@ -165,6 +167,7 @@ public class NonSyncDataInputBuffer exte
     return offset;
   }
 
+  @Override
   public final char readChar() throws IOException {
     if (readToBuff(2) < 0) {
       throw new EOFException();
@@ -182,6 +185,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final double readDouble() throws IOException {
     return Double.longBitsToDouble(readLong());
   }
@@ -195,6 +199,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final float readFloat() throws IOException {
     return Float.intBitsToFloat(readInt());
   }
@@ -211,6 +216,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final void readFully(byte[] buffer) throws IOException {
     readFully(buffer, 0, buffer.length);
   }
@@ -233,6 +239,7 @@ public class NonSyncDataInputBuffer exte
    *           if reaches the end of the stream before enough bytes have been
    *           read
    */
+  @Override
   public final void readFully(byte[] buffer, int offset, int length)
       throws IOException {
     if (length < 0) {
@@ -267,6 +274,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final int readInt() throws IOException {
     if (readToBuff(4) < 0) {
       throw new EOFException();
@@ -292,6 +300,7 @@ public class NonSyncDataInputBuffer exte
    * @deprecated Use BufferedReader
    */
   @Deprecated
+  @Override
   public final String readLine() throws IOException {
     StringBuilder line = new StringBuilder(80); // Typical line length
     boolean foundTerminator = false;
@@ -335,6 +344,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final long readLong() throws IOException {
     if (readToBuff(8) < 0) {
       throw new EOFException();
@@ -356,6 +366,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final short readShort() throws IOException {
     if (readToBuff(2) < 0) {
       throw new EOFException();
@@ -373,6 +384,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final int readUnsignedByte() throws IOException {
     int temp = in.read();
     if (temp < 0) {
@@ -391,6 +403,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final int readUnsignedShort() throws IOException {
     if (readToBuff(2) < 0) {
       throw new EOFException();
@@ -407,6 +420,7 @@ public class NonSyncDataInputBuffer exte
    *           If a problem occurs reading from this DataInputStream.
    * 
    */
+  @Override
   public final String readUTF() throws IOException {
     return decodeUTF(readUnsignedShort());
   }
@@ -450,6 +464,7 @@ public class NonSyncDataInputBuffer exte
    * @throws IOException
    *           If the stream is already closed or another IOException occurs.
    */
+  @Override
   public final int skipBytes(int count) throws IOException {
     int skipped = 0;
     long skip;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java Tue Jul  9 09:07:35 2013
@@ -34,6 +34,8 @@ class OutStream extends PositionedOutput
   static final int HEADER_SIZE = 3;
   private final String name;
   private final OutputReceiver receiver;
+  // if enabled the stream will be suppressed when writing stripe
+  private boolean suppress;
 
   /**
    * Stores the uncompressed bytes that have been serialized, but not
@@ -70,6 +72,7 @@ class OutStream extends PositionedOutput
     this.bufferSize = bufferSize;
     this.codec = codec;
     this.receiver = receiver;
+    this.suppress = false;
   }
 
   public void clear() throws IOException {
@@ -78,6 +81,7 @@ class OutStream extends PositionedOutput
     compressed = null;
     overflow = null;
     current = null;
+    suppress = false;
   }
 
   /**
@@ -264,5 +268,20 @@ class OutStream extends PositionedOutput
     }
     return result;
   }
+
+  /**
+   * Set suppress flag
+   */
+  public void suppress() {
+    suppress = true;
+  }
+
+  /**
+   * Returns the state of suppress flag
+   * @return value of suppress flag
+   */
+  public boolean isSuppressed() {
+    return suppress;
+  }
 }
 

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Tue Jul  9 09:07:35 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FSDataOutput
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
 import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -231,6 +232,14 @@ class WriterImpl implements Writer, Memo
     }
 
     /**
+     * Check the state of suppress flag in output stream
+     * @return value of suppress flag
+     */
+    public boolean isSuppressed() {
+      return outStream.isSuppressed();
+    }
+
+    /**
      * Write the saved compressed buffers to the OutputStream.
      * @param out the stream to write to
      * @throws IOException
@@ -291,9 +300,9 @@ class WriterImpl implements Writer, Memo
      * @return The output outStream that the section needs to be written to.
      * @throws IOException
      */
-    public PositionedOutputStream createStream(int column,
-                                               OrcProto.Stream.Kind kind
-                                              ) throws IOException {
+    public OutStream createStream(int column,
+                                  OrcProto.Stream.Kind kind
+                                  ) throws IOException {
       StreamName name = new StreamName(column, kind);
       BufferedStream result = streams.get(name);
       if (result == null) {
@@ -325,6 +334,14 @@ class WriterImpl implements Writer, Memo
     public boolean buildIndex() {
       return buildIndex;
     }
+
+    /**
+     * Is the ORC file compressed?
+     * @return are the streams compressed
+     */
+    public boolean isCompressed() {
+      return codec != null;
+    }
   }
 
   /**
@@ -337,6 +354,7 @@ class WriterImpl implements Writer, Memo
     protected final int id;
     protected final ObjectInspector inspector;
     private final BitFieldWriter isPresent;
+    private final boolean isCompressed;
     protected final ColumnStatisticsImpl indexStatistics;
     private final ColumnStatisticsImpl fileStatistics;
     protected TreeWriter[] childrenWriters;
@@ -344,6 +362,8 @@ class WriterImpl implements Writer, Memo
     private final OrcProto.RowIndex.Builder rowIndex;
     private final OrcProto.RowIndexEntry.Builder rowIndexEntry;
     private final PositionedOutputStream rowIndexStream;
+    private boolean foundNulls;
+    private OutStream isPresentOutStream;
 
     /**
      * Create a tree writer.
@@ -356,14 +376,17 @@ class WriterImpl implements Writer, Memo
     TreeWriter(int columnId, ObjectInspector inspector,
                StreamFactory streamFactory,
                boolean nullable) throws IOException {
+      this.isCompressed = streamFactory.isCompressed();
       this.id = columnId;
       this.inspector = inspector;
       if (nullable) {
-        isPresent = new BitFieldWriter(streamFactory.createStream(id,
-            OrcProto.Stream.Kind.PRESENT), 1);
+        isPresentOutStream = streamFactory.createStream(id,
+            OrcProto.Stream.Kind.PRESENT);
+        isPresent = new BitFieldWriter(isPresentOutStream, 1);
       } else {
         isPresent = null;
       }
+      this.foundNulls = false;
       indexStatistics = ColumnStatisticsImpl.create(inspector);
       fileStatistics = ColumnStatisticsImpl.create(inspector);
       childrenWriters = new TreeWriter[0];
@@ -401,6 +424,20 @@ class WriterImpl implements Writer, Memo
       }
       if (isPresent != null) {
         isPresent.write(obj == null ? 0 : 1);
+        if(obj == null) {
+          foundNulls = true;
+        }
+      }
+    }
+
+    private void removeIsPresentPositions() {
+      for(int i=0; i < rowIndex.getEntryCount(); ++i) {
+        RowIndexEntry.Builder entry = rowIndex.getEntryBuilder(i);
+        List<Long> positions = entry.getPositionsList();
+        // bit streams use 3 positions if uncompressed, 4 if compressed
+        positions = positions.subList(isCompressed ? 4 : 3, positions.size());
+        entry.clearPositions();
+        entry.addAllPositions(positions);
       }
     }
 
@@ -418,7 +455,21 @@ class WriterImpl implements Writer, Memo
                      int requiredIndexEntries) throws IOException {
       if (isPresent != null) {
         isPresent.flush();
+
+        // if no nulls are found in a stream, then suppress the stream
+        if(!foundNulls) {
+          isPresentOutStream.suppress();
+          // since isPresent bitstream is suppressed, update the index to
+          // remove the positions of the isPresent stream
+          if (rowIndexStream != null) {
+            removeIsPresentPositions();
+          }
+        }
       }
+
+      // reset the flag for next stripe
+      foundNulls = false;
+
       builder.addColumns(getEncoding());
       if (rowIndexStream != null) {
         if (rowIndex.getEntryCount() != requiredIndexEntries) {
@@ -810,6 +861,7 @@ class WriterImpl implements Writer, Memo
      * and augments them with the final information as the stripe is written.
      * @throws IOException
      */
+    @Override
     void createRowIndexEntry() throws IOException {
       getFileStatistics().merge(indexStatistics);
       OrcProto.RowIndexEntry.Builder rowIndexEntry = getRowIndexEntry();
@@ -1349,19 +1401,21 @@ class WriterImpl implements Writer, Memo
       long indexEnd = start;
       for(Map.Entry<StreamName, BufferedStream> pair: streams.entrySet()) {
         BufferedStream stream = pair.getValue();
-        stream.flush();
-        stream.spillTo(rawWriter);
-        stream.clear();
-        long end = rawWriter.getPos();
-        StreamName name = pair.getKey();
-        builder.addStreams(OrcProto.Stream.newBuilder()
-            .setColumn(name.getColumn())
-            .setKind(name.getKind())
-            .setLength(end-section));
-        section = end;
-        if (StreamName.Area.INDEX == name.getArea()) {
-          indexEnd = end;
+        if (!stream.isSuppressed()) {
+          stream.flush();
+          stream.spillTo(rawWriter);
+          long end = rawWriter.getPos();
+          StreamName name = pair.getKey();
+          builder.addStreams(OrcProto.Stream.newBuilder()
+              .setColumn(name.getColumn())
+              .setKind(name.getKind())
+              .setLength(end-section));
+          section = end;
+          if (StreamName.Area.INDEX == name.getArea()) {
+            indexEnd = end;
+          }
         }
+        stream.clear();
       }
       builder.build().writeTo(protobufWriter);
       protobufWriter.flush();

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java Tue Jul  9 09:07:35 2013
@@ -378,10 +378,4 @@ public class BlockMergeTask extends Task
   public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
     // no op
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no op
-  }
-
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java Tue Jul  9 09:07:35 2013
@@ -379,11 +379,4 @@ public class PartialScanTask extends Tas
   public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
     // no op
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no op
-  }
-
-
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java Tue Jul  9 09:07:35 2013
@@ -254,10 +254,4 @@ public class ColumnTruncateTask extends 
   public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
     // no op
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no op
-  }
-
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java Tue Jul  9 09:07:35 2013
@@ -60,6 +60,7 @@ public class DefaultRuleDispatcher imple
    *          the operators encountered so far
    * @throws SemanticException
    */
+  @Override
   public Object dispatch(Node nd, Stack<Node> ndStack, Object... nodeOutputs)
       throws SemanticException {
 

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java Tue Jul  9 09:07:35 2013
@@ -58,9 +58,10 @@ public class RuleRegExp implements Rule 
    * @return cost of the function
    * @throws SemanticException
    */
+  @Override
   public int cost(Stack<Node> stack) throws SemanticException {
     int numElems = (stack != null ? stack.size() : 0);
-    String name = new String();
+    String name = "";
     for (int pos = numElems - 1; pos >= 0; pos--) {
       name = stack.get(pos).getName() + "%" + name;
       Matcher m = pattern.matcher(name);
@@ -68,13 +69,13 @@ public class RuleRegExp implements Rule 
         return m.group().length();
       }
     }
-
     return -1;
   }
 
   /**
    * @return the name of the Node
    **/
+  @Override
   public String getName() {
     return ruleName;
   }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java Tue Jul  9 09:07:35 2013
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hive.ql.lockmgr;
 
+import java.util.Arrays;
+
 import org.apache.hadoop.hive.ql.metadata.DummyPartition;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
@@ -94,16 +96,16 @@ public class HiveLockObject {
       }
 
       HiveLockObjectData target = (HiveLockObjectData) o;
-      boolean ret = (queryId == null) ? target.getQueryId() == null :
-          queryId.equals(target.getQueryId());
-      ret = ret && (lockTime == null) ? target.getLockTime() == null :
-          queryId.equals(target.getLockTime());
-      ret = ret && (lockMode == null) ? target.getLockMode() == null :
-          queryId.equals(target.getLockMode());
-      ret = ret && (queryStr == null) ? target.getQueryStr() == null :
-          queryStr.equals(target.getQueryStr());
-      ret = ret && (clientIp == null) ? target.getClientIp() == null :
-          clientIp.equals(target.getClientIp());
+      boolean ret = (queryId == null ? target.queryId == null :
+          target.queryId != null && queryId.equals(target.queryId));
+      ret = ret && (lockTime == null ? target.lockTime == null :
+          target.lockTime != null && lockTime.equals(target.lockTime));
+      ret = ret && (lockMode == null ? target.lockMode == null :
+          target.lockMode != null && lockMode.equals(target.lockMode));
+      ret = ret && (queryStr == null ? target.queryStr == null :
+          target.queryStr != null && queryStr.equals(target.queryStr));
+      ret = ret && (clientIp == null ? target.clientIp == null :
+          target.clientIp != null && clientIp.equals(target.clientIp));
 
       return ret;
     }
@@ -145,20 +147,17 @@ public class HiveLockObject {
   }
 
   public String getName() {
-    if (this.pathNames == null) {
+    if (pathNames == null) {
       return null;
     }
-    String ret = "";
-    boolean first = true;
+    StringBuilder builder = new StringBuilder();
     for (int i = 0; i < pathNames.length; i++) {
-      if (!first) {
-        ret = ret + "/";
-      } else {
-        first = false;
+      if (i > 0) {
+        builder.append('/');
       }
-      ret = ret + pathNames[i];
+      builder.append(pathNames[i]);
     }
-    return ret;
+    return builder.toString();
   }
 
   public String getDisplayName() {
@@ -200,7 +199,8 @@ public class HiveLockObject {
     }
 
     HiveLockObject tgt = (HiveLockObject) o;
-    return getName().equals(tgt.getName()) &&
-        data == null ? tgt.getData() == null : data.equals(tgt.getData());
+    return Arrays.equals(pathNames, tgt.pathNames) &&
+        data == null ? tgt.getData() == null :
+        tgt.getData() != null && data.equals(tgt.getData());
   }
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java Tue Jul  9 09:07:35 2013
@@ -55,6 +55,8 @@ import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooKeeper;
 
+import com.google.common.annotations.VisibleForTesting;
+
 public class ZooKeeperHiveLockManager implements HiveLockManager {
   HiveLockManagerCtx ctx;
   public static final Log LOG = LogFactory.getLog("ZooKeeperHiveLockManager");
@@ -439,20 +441,25 @@ public class ZooKeeperHiveLockManager im
   }
 
   /* Remove the lock specified */
-  private static void unlockPrimitive(HiveConf conf, ZooKeeper zkpClient,
+  @VisibleForTesting
+  static void unlockPrimitive(HiveConf conf, ZooKeeper zkpClient,
                              HiveLock hiveLock, String parent) throws LockException {
     ZooKeeperHiveLock zLock = (ZooKeeperHiveLock)hiveLock;
     try {
+      // can throw KeeperException.NoNodeException, which might mean something is wrong
       zkpClient.delete(zLock.getPath(), -1);
 
       // Delete the parent node if all the children have been deleted
       HiveLockObject obj = zLock.getHiveLockObject();
       String name  = getLastObjectName(parent, obj);
 
-      List<String> children = zkpClient.getChildren(name, false);
-      if ((children == null) || (children.isEmpty()))
-      {
-        zkpClient.delete(name, -1);
+      try {
+        List<String> children = zkpClient.getChildren(name, false);
+        if (children == null || children.isEmpty()) {
+          zkpClient.delete(name, -1);
+        }
+      } catch (KeeperException.NoNodeException e) {
+        LOG.debug("Node " + name + " previously deleted when attempting to delete.");
       }
     } catch (Exception e) {
       LOG.error("Failed to release ZooKeeper lock: ", e);

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Jul  9 09:07:35 2013
@@ -729,8 +729,9 @@ public class Hive {
       List<Order> sortCols = new ArrayList<Order>();
       storageDescriptor.setBucketCols(null);
       int k = 0;
-      for (int i = 0; i < storageDescriptor.getCols().size(); i++) {
-        FieldSchema col = storageDescriptor.getCols().get(i);
+      Table metaBaseTbl = new Table(baseTbl);
+      for (int i = 0; i < metaBaseTbl.getCols().size(); i++) {
+        FieldSchema col = metaBaseTbl.getCols().get(i);
         if (indexedCols.contains(col.getName())) {
           indexTblCols.add(col);
           sortCols.add(new Order(col.getName(), 1));
@@ -950,7 +951,7 @@ public class Hive {
     } catch (NoSuchObjectException e) {
       if (throwException) {
         LOG.error(StringUtils.stringifyException(e));
-        throw new InvalidTableException("Table " + tableName + " not found ", tableName);
+        throw new InvalidTableException(tableName);
       }
       return null;
     } catch (Exception e) {

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveException.java Tue Jul  9 09:07:35 2013
@@ -18,11 +18,17 @@
 
 package org.apache.hadoop.hive.ql.metadata;
 
+import org.apache.hadoop.hive.ql.ErrorMsg;
+
 /**
  * Generic exception class for Hive.
  */
 
 public class HiveException extends Exception {
+  /**
+   * Standard predefined message with error code and possibly SQL State, etc.
+   */
+  private ErrorMsg canonicalErrorMsg = ErrorMsg.GENERIC_ERROR;
   public HiveException() {
     super();
   }
@@ -38,4 +44,26 @@ public class HiveException extends Excep
   public HiveException(String message, Throwable cause) {
     super(message, cause);
   }
+
+  public HiveException(ErrorMsg message, String... msgArgs) {
+    this(null, message, msgArgs);
+  }
+
+  /**
+   * This is the recommended constructor to use since it helps use
+   * canonical messages throughout.  
+   * @param errorMsg Canonical error message
+   * @param msgArgs message arguments if message is parametrized; must be {@code null} is message takes no arguments
+   */
+  public HiveException(Throwable cause, ErrorMsg errorMsg, String... msgArgs) {
+    super(errorMsg.format(msgArgs), cause);
+    canonicalErrorMsg = errorMsg;
+
+  }
+  /**
+   * @return {@link ErrorMsg#GENERIC_ERROR} by default
+   */
+  public ErrorMsg getCanonicalErrorMsg() {
+    return canonicalErrorMsg;
+  }
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/InvalidTableException.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/InvalidTableException.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/InvalidTableException.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/InvalidTableException.java Tue Jul  9 09:07:35 2013
@@ -18,31 +18,17 @@
 
 package org.apache.hadoop.hive.ql.metadata;
 
+import org.apache.hadoop.hive.ql.ErrorMsg;
+
 /**
  * Generic exception class for Hive.
- * 
  */
 
 public class InvalidTableException extends HiveException {
   String tableName;
 
   public InvalidTableException(String tableName) {
-    super();
-    this.tableName = tableName;
-  }
-
-  public InvalidTableException(String message, String tableName) {
-    super(message);
-    this.tableName = tableName;
-  }
-
-  public InvalidTableException(Throwable cause, String tableName) {
-    super(cause);
-    this.tableName = tableName;
-  }
-
-  public InvalidTableException(String message, Throwable cause, String tableName) {
-    super(message, cause);
+    super(ErrorMsg.INVALID_TABLE, tableName);
     this.tableName = tableName;
   }
 

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java Tue Jul  9 09:07:35 2013
@@ -48,12 +48,12 @@ import org.codehaus.jackson.map.ObjectMa
  * json.
  */
 public class JsonMetaDataFormatter implements MetaDataFormatter {
-    private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask");
+    private static final Log LOG = LogFactory.getLog(JsonMetaDataFormatter.class);
 
     /**
      * Convert the map to a JSON string.
      */
-    public void asJson(OutputStream out, Map<String, Object> data)
+    private void asJson(OutputStream out, Map<String, Object> data)
         throws HiveException
     {
         try {
@@ -66,70 +66,29 @@ public class JsonMetaDataFormatter imple
     /**
      * Write an error message.
      */
-    public void error(OutputStream out, String msg, int errorCode)
+    @Override
+    public void error(OutputStream out, String msg, int errorCode, String sqlState)
         throws HiveException
     {
-        asJson(out,
-               MapBuilder.create()
-               .put("error", msg)
-               .put("errorCode", errorCode)
-               .build());
+        error(out, msg, errorCode, sqlState, null);
     }
-
-    /**
-     * Write a log warn message.
-     */
-    public void logWarn(OutputStream out, String msg, int errorCode)
-        throws HiveException
-    {
-        LOG.warn(msg);
-        error(out, msg, errorCode);
-    }
-
-    /**
-     * Write a log info message.
-     */
-    public void logInfo(OutputStream out, String msg, int errorCode)
-        throws HiveException
-    {
-        LOG.info(msg);
-        error(out, msg, errorCode);
-    }
-
-    /**
-     * Write a console error message.
-     */
-    public void consoleError(LogHelper console, String msg, int errorCode) {
-        try {
-            console.printError(msg);
-            error(console.getOutStream(), msg, errorCode);
-        } catch (HiveException e) {
-            console.printError("unable to create json: " + e);
+    @Override
+    public void error(OutputStream out, String errorMessage, int errorCode, String sqlState, String errorDetail) throws HiveException {
+        MapBuilder mb = MapBuilder.create().put("error", errorMessage);
+        if(errorDetail != null) {
+            mb.put("errorDetail", errorDetail);
         }
-    }
-
-    /**
-     * Write a console error message.
-     */
-    public void consoleError(LogHelper console, String msg, String detail,
-                             int errorCode)
-    {
-        try {
-            console.printError(msg, detail);
-            asJson(console.getOutStream(),
-                   MapBuilder.create()
-                   .put("error", msg)
-                   .put("errorDetail", detail)
-                   .put("errorCode", errorCode)
-                   .build());
-        } catch (HiveException e) {
-            console.printError("unable to create json: " + e);
+        mb.put("errorCode", errorCode);
+        if(sqlState != null) {
+          mb.put("sqlState", sqlState);
         }
+        asJson(out,mb.build());
     }
 
     /**
      * Show a list of tables.
      */
+    @Override
     public void showTables(DataOutputStream out, Set<String> tables)
         throws HiveException
     {
@@ -142,6 +101,7 @@ public class JsonMetaDataFormatter imple
     /**
      * Describe table.
      */
+    @Override
     public void describeTable(DataOutputStream out,
                               String colPath, String tableName,
                               Table tbl, Partition part, List<FieldSchema> cols,
@@ -178,6 +138,7 @@ public class JsonMetaDataFormatter imple
             .build();
     }
 
+    @Override
     public void showTableStatus(DataOutputStream out,
                                 Hive db,
                                 HiveConf conf,
@@ -366,6 +327,7 @@ public class JsonMetaDataFormatter imple
     /**
      * Show the table partitions.
      */
+    @Override
     public void showTablePartitons(DataOutputStream out, List<String> parts)
         throws HiveException
     {
@@ -424,6 +386,7 @@ public class JsonMetaDataFormatter imple
     /**
      * Show a list of databases
      */
+    @Override
     public void showDatabases(DataOutputStream out, List<String> databases)
         throws HiveException
     {
@@ -436,6 +399,7 @@ public class JsonMetaDataFormatter imple
     /**
      * Show the description of a database
      */
+    @Override
     public void showDatabaseDescription(DataOutputStream out,
                                         String database,
                                         String comment,

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java Tue Jul  9 09:07:35 2013
@@ -28,6 +28,7 @@ import java.util.Map.Entry;
 import java.util.Set;
 
 import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Index;
@@ -325,4 +326,11 @@ public final class MetaDataFormatUtils {
     formatOutput(ShowIndexesDesc.getSchema().split("#")[0].split(","), indexCols);
     return indexCols.toString();
   }
+  public static MetaDataFormatter getFormatter(HiveConf conf) {
+    if ("json".equals(conf.get(HiveConf.ConfVars.HIVE_DDL_OUTPUT_FORMAT.varname, "text"))) {
+      return new JsonMetaDataFormatter();
+    } else {
+      return new TextMetaDataFormatter(conf.getIntVar(HiveConf.ConfVars.CLIPRETTYOUTPUTNUMCOLS));
+    }
+  }
 }

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java Tue Jul  9 09:07:35 2013
@@ -38,49 +38,18 @@ import org.apache.hadoop.hive.ql.session
  */
 public interface MetaDataFormatter {
     /**
-     * Generic error code.  This and the other error codes are
-     * designed to match the HTTP status codes.
-     */
-    static final int ERROR = 500;
-
-    /**
-     * Missing error code.
-     */
-    static final int MISSING = 404;
-
-    /**
-     * Conflict error code.
-     */
-    static final int CONFLICT = 409;
-
-    /**
      * Write an error message.
+     * @param sqlState if {@code null}, will be ignored
      */
-    public void error(OutputStream out, String msg, int errorCode)
-        throws HiveException;
-
-    /**
-     * Write a log warn message.
-     */
-    public void logWarn(OutputStream out, String msg, int errorCode)
-        throws HiveException;
-
-    /**
-     * Write a log info message.
-     */
-    public void logInfo(OutputStream out, String msg, int errorCode)
+    public void error(OutputStream out, String msg, int errorCode, String sqlState)
         throws HiveException;
 
-    /**
-     * Write a console error message.
-     */
-    public void consoleError(LogHelper console, String msg, int errorCode);
-
-    /**
-     * Write a console error message.
-     */
-    public void consoleError(LogHelper console, String msg, String detail,
-                             int errorCode);
+  /**
+   * @param sqlState if {@code null}, will be skipped in output
+   * @param errorDetail usually string version of some Exception, if {@code null}, will be ignored
+   */
+    public void error(OutputStream out, String errorMessage, int errorCode, String sqlState, String errorDetail)
+          throws HiveException;
 
     /**
      * Show a list of tables.

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java Tue Jul  9 09:07:35 2013
@@ -49,8 +49,8 @@ import org.apache.hadoop.hive.shims.Shim
  * Format table and index information for human readability using
  * simple lines of text.
  */
-public class TextMetaDataFormatter implements MetaDataFormatter {
-    private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask");
+class TextMetaDataFormatter implements MetaDataFormatter {
+    private static final Log LOG = LogFactory.getLog(TextMetaDataFormatter.class);
 
     private static final int separator = Utilities.tabCode;
     private static final int terminator = Utilities.newLineCode;
@@ -67,54 +67,35 @@ public class TextMetaDataFormatter imple
     /**
      * Write an error message.
      */
-    public void error(OutputStream out, String msg, int errorCode)
+    @Override
+    public void error(OutputStream out, String msg, int errorCode, String sqlState)
         throws HiveException
     {
-        try {
-            out.write(msg.getBytes("UTF-8"));
-            out.write(terminator);
-        } catch (Exception e) {
-            throw new HiveException(e);
-        }
-    }
-
-    /**
-     * Write a log warn message.
-     */
-    public void logWarn(OutputStream out, String msg, int errorCode)
-        throws HiveException
-    {
-        LOG.warn(msg);
+        error(out, msg, errorCode, sqlState, null);
     }
 
-    /**
-     * Write a log info message.
-     */
-    public void logInfo(OutputStream out, String msg, int errorCode)
-        throws HiveException
+    @Override
+    public void error(OutputStream out, String errorMessage, int errorCode, String sqlState, String errorDetail)
+          throws HiveException
     {
-        LOG.info(msg);
-    }
-
-    /**
-     * Write a console error message.
-     */
-    public void consoleError(LogHelper console, String msg, int errorCode) {
-        console.printError(msg);
-    }
-
-    /**
-     * Write a console error message.
-     */
-    public void consoleError(LogHelper console, String msg, String detail,
-                             int errorCode)
-    {
-        console.printError(msg, detail);
+      try {
+        out.write(errorMessage.getBytes("UTF-8"));
+        if(errorDetail != null) {
+          out.write(errorDetail.getBytes("UTF-8"));
+        }
+        out.write(errorCode);
+        if(sqlState != null) {
+          out.write(sqlState.getBytes("UTF-8"));//this breaks all the tests in .q files
+        }
+        out.write(terminator);
+      } catch (Exception e) {
+          throw new HiveException(e);
+        }
     }
-
     /**
      * Show a list of tables.
      */
+    @Override
     public void showTables(DataOutputStream out, Set<String> tables)
         throws HiveException
     {
@@ -131,6 +112,7 @@ public class TextMetaDataFormatter imple
         }
     }
 
+    @Override
     public void describeTable(DataOutputStream outStream,
                               String colPath, String tableName,
                               Table tbl, Partition part, List<FieldSchema> cols,
@@ -187,6 +169,7 @@ public class TextMetaDataFormatter imple
         }
     }
 
+    @Override
     public void showTableStatus(DataOutputStream outStream,
                                 Hive db,
                                 HiveConf conf,
@@ -406,6 +389,7 @@ public class TextMetaDataFormatter imple
     /**
      * Show the table partitions.
      */
+    @Override
     public void showTablePartitons(DataOutputStream outStream, List<String> parts)
         throws HiveException
     {
@@ -430,6 +414,7 @@ public class TextMetaDataFormatter imple
     /**
      * Show the list of databases
      */
+    @Override
     public void showDatabases(DataOutputStream outStream, List<String> databases)
         throws HiveException
         {
@@ -447,6 +432,7 @@ public class TextMetaDataFormatter imple
     /**
      * Describe a database
      */
+    @Override
     public void showDatabaseDescription(DataOutputStream outStream,
                                         String database,
                                         String comment,



Mime
View raw message