hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1552041 [2/7] - in /hive/branches/tez: ./ ant/src/org/apache/hadoop/hive/ant/ conf/ hbase-handler/src/java/org/apache/hadoop/hive/hbase/ hcatalog/src/test/e2e/templeton/tests/ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/tem...
Date Wed, 18 Dec 2013 17:22:05 GMT
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java Wed Dec 18 17:22:02 2013
@@ -25,6 +25,8 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.plan.DummyStoreDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 
 /**
  * For SortMerge joins, this is a dummy operator, which stores the row for the
@@ -71,15 +73,28 @@ public class DummyStoreOperator extends 
 
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {
-    outputObjInspector = inputObjInspectors[0];
+    /*
+     * The conversion to standard object inspector was necessitated by HIVE-5973. The issue 
+     * happens when a select operator preceeds this operator as in the case of a subquery. The 
+     * select operator does not allocate a new object to hold the deserialized row. This affects 
+     * the operation of the SMB join which puts the object in a priority queue. Since all elements
+     * of the priority queue point to the same object, the join was resulting in incorrect 
+     * results.
+     *
+     * So the fix is to make a copy of the object as done in the processOp phase below. This
+     * however necessitates a change in the object inspector that can be used in processing the
+     * row downstream.
+     */
+    outputObjInspector = ObjectInspectorUtils.getStandardObjectInspector(inputObjInspectors[0]);
     result = new InspectableObject(null, outputObjInspector);
     initializeChildren(hconf);
   }
 
   @Override
   public void processOp(Object row, int tag) throws HiveException {
-    // Store the row
-    result.o = row;
+    // Store the row. See comments above for why we need a new copy of the row.
+    result.o = ObjectInspectorUtils.copyToStandardObject(row, inputObjInspectors[0],
+        ObjectInspectorCopyOption.WRITABLE);
   }
 
   @Override

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Wed Dec 18 17:22:02 2013
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec;
 
 import java.io.IOException;
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -122,14 +121,13 @@ public class FetchTask extends Task<Fetc
     this.maxRows = maxRows;
   }
 
-  @Override
-  public boolean fetch(ArrayList<String> res) throws IOException, CommandNeedRetryException {
+  public boolean fetch(List res) throws IOException, CommandNeedRetryException {
     sink.reset(res);
+    int rowsRet = work.getLeastNumRows();
+    if (rowsRet <= 0) {
+      rowsRet = work.getLimit() >= 0 ? Math.min(work.getLimit() - totalRows, maxRows) : maxRows;
+    }
     try {
-      int rowsRet = work.getLeastNumRows();
-      if (rowsRet <= 0) {
-        rowsRet = work.getLimit() >= 0 ? Math.min(work.getLimit() - totalRows, maxRows) : maxRows;
-      }
       if (rowsRet <= 0) {
         fetch.clearFetchContext();
         return false;

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Wed Dec 18 17:22:02 2013
@@ -900,35 +900,36 @@ public class FileSinkOperator extends Te
     }
 
     String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(hconf));
-    String spSpec = conf.getStaticSpec() != null ? conf.getStaticSpec() : "";
+    String spSpec = conf.getStaticSpec();
 
-    for (String fspKey : valToPaths.keySet()) {
-      FSPaths fspValue = valToPaths.get(fspKey);
-      String key;
-
-      // construct the key(fileID) to insert into the intermediate stats table
-      if (fspKey == "") {
-        if (statsPublisher instanceof CounterStatsPublisher) {
-          // key is of form either of dbName.TblName/ or dbName.TblName/p1=v1/
-          key = Utilities.appendPathSeparator(conf.getTableInfo().getTableName() + Path.SEPARATOR + spSpec);
-          } else {
-        // for non-partitioned/static partitioned table, the key for temp storage is
-        // common key prefix + static partition spec + taskID
-        String keyPrefix = Utilities.getHashedStatsPrefix(
-            conf.getStatsAggPrefix() + spSpec, conf.getMaxStatsKeyPrefixLength());
-        key = keyPrefix + taskID;
-          }
+    int maxKeyLength = conf.getMaxStatsKeyPrefixLength();
+    boolean counterStats = statsPublisher instanceof CounterStatsPublisher;
+
+    for (Map.Entry<String, FSPaths> entry : valToPaths.entrySet()) {
+      String fspKey = entry.getKey();     // DP/LB
+      FSPaths fspValue = entry.getValue();
+
+      // split[0] = DP, split[1] = LB
+      String[] split = splitKey(fspKey);
+      String dpSpec = split[0];
+      String lbSpec = split[1];
+
+      String prefix;
+      String postfix;
+      if (counterStats) {
+        // key = "database.table/SP/DP/"LB/
+        prefix = conf.getTableInfo().getTableName();
+        postfix = Utilities.join(lbSpec);
       } else {
-          if (statsPublisher instanceof CounterStatsPublisher) {
-          // key is of form either of dbName.TblName/p1=v1/
-            key = Utilities.appendPathSeparator(Utilities.appendPathSeparator(
-              conf.getTableInfo().getTableName() + Path.SEPARATOR + spSpec) + fspKey);
-          } else {
-              // for partitioned table, the key is
-              // common key prefix + static partition spec + DynamicPartSpec + taskID
-              key = createKeyForStatsPublisher(taskID, spSpec, fspKey);
-          }
+        // key = "prefix/SP/DP/"LB/taskID/
+        prefix = conf.getStatsAggPrefix();
+        postfix = Utilities.join(lbSpec, taskID);
       }
+      prefix = Utilities.join(prefix, spSpec, dpSpec);
+      prefix = Utilities.getHashedStatsPrefix(prefix, maxKeyLength, postfix.length());
+
+      String key = Utilities.join(prefix, postfix);
+
       Map<String, String> statsToPublish = new HashMap<String, String>();
       for (String statType : fspValue.stat.getStoredStats()) {
         statsToPublish.put(statType, Long.toString(fspValue.stat.getStat(statType)));
@@ -984,29 +985,20 @@ public class FileSinkOperator extends Te
    * key=484/value=val_484 or
    * HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME/HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME
    * so, at the end, "keyPrefix" doesn't have subdir information from skewed but "key" has
-   * @param taskID
-   * @param spSpec
-   * @param fspKey
-   * @return
+   *
+   * In a word, fspKey is consists of DP(dynamic partition spec) + LB(list bucketing spec)
+   * In stats publishing, full partition spec consists of prefix part of stat key
+   * but list bucketing spec is regarded as a postfix of stat key. So we split it here.
    */
-  private String createKeyForStatsPublisher(String taskID, String spSpec, String fspKey) {
-    String key;
-    String newFspKey = fspKey;
-    String storedAsDirPostFix = "";
-    if (isSkewedStoredAsSubDirectories) {
-      List<String> skewedValueDirList = this.lbCtx.getSkewedValuesDirNames();
-      for (String dir : skewedValueDirList) {
-        newFspKey = newFspKey.replace(dir, "");
-        if (!newFspKey.equals(fspKey)) {
-          storedAsDirPostFix = dir;
-          break;
+  private String[] splitKey(String fspKey) {
+    if (!fspKey.isEmpty() && isSkewedStoredAsSubDirectories) {
+      for (String dir : lbCtx.getSkewedValuesDirNames()) {
+        int index = fspKey.indexOf(dir);
+        if (index >= 0) {
+          return new String[] {fspKey.substring(0, index), fspKey.substring(index + 1)};
         }
       }
     }
-    String keyPrefix = Utilities.getHashedStatsPrefix(
-        conf.getStatsAggPrefix() + spSpec + newFspKey,
-        conf.getMaxStatsKeyPrefixLength());
-    key = keyPrefix + storedAsDirPostFix + taskID;
-    return key;
+    return new String[] {fspKey, null};
   }
 }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java Wed Dec 18 17:22:02 2013
@@ -18,19 +18,15 @@
 
 package org.apache.hadoop.hive.ql.exec;
 
-import java.util.ArrayList;
+import java.util.List;
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.ListSinkDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.DelimitedJSONSerDe;
-import org.apache.hadoop.hive.serde2.SerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.util.ReflectionUtils;
 
 /**
@@ -39,45 +35,43 @@ import org.apache.hadoop.util.Reflection
  */
 public class ListSinkOperator extends Operator<ListSinkDesc> {
 
-  private transient SerDe mSerde;
+  public static final String OUTPUT_FORMATTER = "output.formatter";
 
-  private transient ArrayList<String> res;
+  private transient List res;
+  private transient FetchFormatter fetcher;
   private transient int numRows;
 
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {
     try {
-      mSerde = initializeSerde(hconf);
-      initializeChildren(hconf);
+      fetcher = initializeFetcher(hconf);
     } catch (Exception e) {
       throw new HiveException(e);
     }
+    super.initializeOp(hconf);
   }
 
-  private SerDe initializeSerde(Configuration conf) throws Exception {
-    String serdeName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE);
-    Class<? extends SerDe> serdeClass = Class.forName(serdeName, true,
-        JavaUtils.getClassLoader()).asSubclass(SerDe.class);
-    // cast only needed for Hadoop 0.17 compatibility
-    SerDe serde = ReflectionUtils.newInstance(serdeClass, null);
-
-    Properties serdeProp = new Properties();
-
-    // this is the default serialization format
-    if (serde instanceof DelimitedJSONSerDe) {
-      serdeProp.put(serdeConstants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
-      serdeProp.put(serdeConstants.SERIALIZATION_NULL_FORMAT, getConf().getSerializationNullFormat());
+  private FetchFormatter initializeFetcher(Configuration conf) throws Exception {
+    String formatterName = conf.get(OUTPUT_FORMATTER);
+    FetchFormatter fetcher;
+    if (formatterName != null && !formatterName.isEmpty()) {
+      Class<? extends FetchFormatter> fetcherClass = Class.forName(formatterName, true,
+          JavaUtils.getClassLoader()).asSubclass(FetchFormatter.class);
+      fetcher = ReflectionUtils.newInstance(fetcherClass, null);
+    } else {
+      fetcher = new DefaultFetchFormatter();
     }
-    serde.initialize(conf, serdeProp);
-    return serde;
-  }
 
-  public ListSinkOperator initialize(SerDe mSerde) {
-    this.mSerde = mSerde;
-    return this;
+    // selectively used by fetch formatter
+    Properties props = new Properties();
+    props.put(serdeConstants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
+    props.put(serdeConstants.SERIALIZATION_NULL_FORMAT, getConf().getSerializationNullFormat());
+
+    fetcher.initialize(conf, props);
+    return fetcher;
   }
 
-  public void reset(ArrayList<String> res) {
+  public void reset(List res) {
     this.res = res;
     this.numRows = 0;
   }
@@ -86,11 +80,12 @@ public class ListSinkOperator extends Op
     return numRows;
   }
 
+  @SuppressWarnings("unchecked")
   public void processOp(Object row, int tag) throws HiveException {
     try {
-      res.add(mSerde.serialize(row, outputObjInspector).toString());
+      res.add(fetcher.convert(row, inputObjInspectors[0]));
       numRows++;
-    } catch (SerDeException e) {
+    } catch (Exception e) {
       throw new HiveException(e);
     }
   }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Wed Dec 18 17:22:02 2013
@@ -467,6 +467,10 @@ public abstract class Operator<T extends
     this.inputObjInspectors = inputObjInspectors;
   }
 
+  public ObjectInspector getOutputObjInspector() {
+    return outputObjInspector;
+  }
+
   /**
    * Process the row.
    *

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Wed Dec 18 17:22:02 2013
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.exec;
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -29,11 +28,11 @@ import java.util.Map;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.ql.DriverContext;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -44,10 +43,10 @@ import org.apache.hadoop.hive.ql.plan.Dy
 import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
 import org.apache.hadoop.hive.ql.plan.StatsWork;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.apache.hadoop.hive.ql.stats.CounterStatsAggregator;
 import org.apache.hadoop.hive.ql.stats.StatsAggregator;
 import org.apache.hadoop.hive.ql.stats.StatsFactory;
 import org.apache.hadoop.hive.ql.stats.StatsPublisher;
-import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.util.StringUtils;
 
 /**
@@ -71,44 +70,6 @@ public class StatsTask extends Task<Stat
     dpPartSpecs = null;
   }
 
-  /**
-   *
-   * Statistics for a Partition or Unpartitioned Table
-   *
-   */
-  class Statistics {
-    Map<String, LongWritable> stats;
-
-    public Statistics() {
-      stats = new HashMap<String, LongWritable>();
-      for (String statType : StatsSetupConst.supportedStats) {
-        stats.put(statType, new LongWritable(0L));
-      }
-    }
-
-    public Statistics(Map<String, Long> st) {
-      stats = new HashMap<String, LongWritable>();
-      for (String statType : st.keySet()) {
-        Long stValue = st.get(statType) == null ? 0L : st.get(statType);
-        stats.put(statType, new LongWritable(stValue));
-      }
-    }
-
-    public long getStat(String statType) {
-      return stats.get(statType) == null ? 0L : stats.get(statType).get();
-    }
-
-    public void setStat(String statType, long value) {
-      stats.put(statType, new LongWritable(value));
-    }
-
-
-    @Override
-    public String toString() {
-      return org.apache.commons.lang.StringUtils.join(StatsSetupConst.supportedStats, ", ");
-    }
-  }
-
   @Override
   protected void receiveFeed(FeedType feedType, Object feedValue) {
     // this method should be called by MoveTask when there are dynamic partitions generated
@@ -175,176 +136,93 @@ public class StatsTask extends Task<Stat
     try {
       // Stats setup:
       Warehouse wh = new Warehouse(conf);
-      String statsImplementationClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS);
-      if (!this.getWork().getNoStatsAggregator()) {
-        StatsFactory factory = StatsFactory.newFactory(statsImplementationClass, conf);
-        if (factory != null && work.isNoScanAnalyzeCommand()){
-          // initialize stats publishing table for noscan which has only stats task
-          // the rest of MR task following stats task initializes it in ExecDriver.java
-          StatsPublisher statsPublisher = factory.getStatsPublisher();
-          if (!statsPublisher.init(conf)) { // creating stats table if not exists
-            if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) {
-              throw
-                new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg());
-            }
-          }
-        }
-        if (factory != null) {
-          statsAggregator = factory.getStatsAggregator();
-          // manufacture a StatsAggregator
-          if (!statsAggregator.connect(conf, getWork().getSourceTask())) {
-            throw new HiveException("StatsAggregator connect failed " + statsImplementationClass);
+      if (!getWork().getNoStatsAggregator() && !getWork().isNoScanAnalyzeCommand()) {
+        try {
+          statsAggregator = createStatsAggregator(conf);
+        } catch (HiveException e) {
+          if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) {
+            throw e;
           }
-        }
-      }
-
-      Statistics tblStats = new Statistics();
-
-      org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable();
-      Map<String, String> parameters = tTable.getParameters();
-
-      boolean tableStatsExist = this.existStats(parameters);
-
-      for (String statType : StatsSetupConst.supportedStats) {
-        if (parameters.containsKey(statType)) {
-          tblStats.setStat(statType, Long.parseLong(parameters.get(statType)));
+          console.printError(ErrorMsg.STATS_SKIPPING_BY_ERROR.getErrorCodedMsg(e.toString()));
         }
       }
 
       List<Partition> partitions = getPartitionsList();
       boolean atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC);
-      int maxPrefixLength = HiveConf.getIntVar(conf,
-          HiveConf.ConfVars.HIVE_STATS_KEY_PREFIX_MAX_LENGTH);
+
       String tableFullName = table.getDbName() + "." + table.getTableName();
+      int maxPrefixLength = StatsFactory.getMaxPrefixLength(conf);
+
+      // "counter" type does not need to collect stats per task
+      boolean counterStat = statsAggregator instanceof CounterStatsAggregator;
       if (partitions == null) {
+        org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable();
+        Map<String, String> parameters = tTable.getParameters();
         // non-partitioned tables:
-        if (!tableStatsExist && atomic) {
+        if (!existStats(parameters) && atomic) {
           return 0;
         }
 
-        if (statsAggregator != null) {
-          String aggKey;
-          if (statsImplementationClass.equals("counter")) {
-            // Key is of the form dbName.tblName/
-            aggKey = tableFullName+Path.SEPARATOR;
-            } else {
-             // In case of a non-partitioned table, the key for stats temporary store is "rootDir"
-             aggKey = Utilities.getHashedStatsPrefix(work.getAggKey(), maxPrefixLength);
-            }
-          updateStats(StatsSetupConst.statsRequireCompute, tblStats, statsAggregator, parameters,
-            aggKey, atomic);
-          statsAggregator.cleanUp(aggKey);
-        }
         // The collectable stats for the aggregator needs to be cleared.
         // For eg. if a file is being loaded, the old number of rows are not valid
-        else if (work.isClearAggregatorStats()) {
-          for (String statType : StatsSetupConst.statsRequireCompute) {
-            if (parameters.containsKey(statType)) {
-              tblStats.setStat(statType, 0L);
-            }
-          }
+        if (work.isClearAggregatorStats()) {
+          clearStats(parameters);
         }
 
-        // write table stats to metastore
-        parameters = tTable.getParameters();
-        for (String statType : StatsSetupConst.statsRequireCompute) {
-          parameters.put(statType, Long.toString(tblStats.getStat(statType)));
+        if (statsAggregator != null) {
+          String prefix = getAggregationPrefix(counterStat, table, null);
+          String aggKey = Utilities.getHashedStatsPrefix(prefix, maxPrefixLength, 0);
+          updateStats(statsAggregator, parameters, aggKey, atomic);
+          statsAggregator.cleanUp(aggKey);
         }
+
+        updateQuickStats(wh, parameters, tTable.getSd());
+
+        // write table stats to metastore
         parameters.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
-        tTable.setParameters(parameters);
+
         db.alterTable(tableFullName, new Table(tTable));
 
-        console.printInfo("Table " + tableFullName + " stats: [" + tblStats.toString() + ']');
+        console.printInfo("Table " + tableFullName + " stats: [" + toString(parameters) + ']');
       } else {
         // Partitioned table:
         // Need to get the old stats of the partition
         // and update the table stats based on the old and new stats.
+        List<Partition> updates = new ArrayList<Partition>();
         for (Partition partn : partitions) {
           //
           // get the old partition stats
           //
           org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition();
-          parameters = tPart.getParameters();
-
-          boolean hasStats = this.existStats(parameters);
-          if (!hasStats && atomic) {
+          Map<String, String> parameters = tPart.getParameters();
+          if (!existStats(parameters) && atomic) {
             continue;
           }
 
-          Map<String, Long> currentValues = new HashMap<String, Long>();
-          for (String statType : StatsSetupConst.supportedStats) {
-            Long val = parameters.containsKey(statType) ? Long.parseLong(parameters.get(statType))
-                : 0L;
-            currentValues.put(statType, val);
-          }
-
-          //
-          // get the new partition stats
-          //
-          Statistics newPartStats = new Statistics();
-          String partitionID;
-
-          if (statsImplementationClass.equals("counter")) {
-            // stat-Agg-key is of form : dbName.tblName/p1=v1/p2=val2/
-        	partitionID = Utilities.appendPathSeparator(tableFullName + Path.SEPARATOR + 
-              Warehouse.makePartPath(partn.getSpec()));
-            // there is no need to aggregate stats in this case, but this should also work.
-            // also check non-partitioned code path.
-          } else {
-              // In that case of a partition, the key for stats temporary store is
-              // "rootDir/[dynamic_partition_specs/]%"
-             partitionID = Utilities.getHashedStatsPrefix(
-             work.getAggKey() + Warehouse.makePartPath(partn.getSpec()), maxPrefixLength);
+          // The collectable stats for the aggregator needs to be cleared.
+          // For eg. if a file is being loaded, the old number of rows are not valid
+          if (work.isClearAggregatorStats()) {
+            clearStats(parameters);
           }
-          LOG.info("Stats aggregator : " + partitionID);
 
           if (statsAggregator != null) {
-            updateStats(StatsSetupConst.statsRequireCompute, newPartStats, statsAggregator,
-                parameters, partitionID, atomic);
-            statsAggregator.cleanUp(partitionID);
-          } else {
-            for (String statType : StatsSetupConst.statsRequireCompute) {
-              // The collectable stats for the aggregator needs to be cleared.
-              // For eg. if a file is being loaded, the old number of rows are not valid
-              if (work.isClearAggregatorStats()) {
-                if (parameters.containsKey(statType)) {
-                  newPartStats.setStat(statType, 0L);
-                }
-              }
-              else {
-                newPartStats.setStat(statType, currentValues.get(statType));
-              }
-            }
-          }
-          /**
-           * calculate fast statistics
-           */
-          FileStatus[] partfileStatus = wh.getFileStatusesForPartition(tPart);
-          newPartStats.setStat(StatsSetupConst.NUM_FILES, partfileStatus.length);
-          long partSize = 0L;
-          for (int i = 0; i < partfileStatus.length; i++) {
-            partSize += partfileStatus[i].getLen();
+            String prefix = getAggregationPrefix(counterStat, table, partn);
+            String aggKey = Utilities.getHashedStatsPrefix(prefix, maxPrefixLength, 0);
+            updateStats(statsAggregator, parameters, aggKey, atomic);
+            statsAggregator.cleanUp(aggKey);
           }
-          newPartStats.setStat(StatsSetupConst.TOTAL_SIZE, partSize);
 
-          //
-          // update the metastore
-          //
-          for (String statType : StatsSetupConst.supportedStats) {
-            long statValue = newPartStats.getStat(statType);
-            if (statValue >= 0) {
-              parameters.put(statType, Long.toString(newPartStats.getStat(statType)));
-            }
-          }
+          updateQuickStats(wh, parameters, tPart.getSd());
 
           parameters.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
-          tPart.setParameters(parameters);
-          db.alterPartition(tableFullName, new Partition(table, tPart));
+          updates.add(new Partition(table, tPart));
 
           console.printInfo("Partition " + tableFullName + partn.getSpec() +
-              " stats: [" + newPartStats.toString() + ']');
+              " stats: [" + toString(parameters) + ']');
+        }
+        if (!updates.isEmpty()) {
+          db.alterPartitions(tableFullName, updates);
         }
-
       }
 
     } catch (Exception e) {
@@ -366,6 +244,49 @@ public class StatsTask extends Task<Stat
     return ret;
   }
 
+  private String getAggregationPrefix(boolean counter, Table table, Partition partition)
+      throws MetaException {
+    if (!counter && partition == null) {
+      return work.getAggKey();
+    }
+    StringBuilder prefix = new StringBuilder();
+    if (counter) {
+      // prefix is of the form dbName.tblName
+      prefix.append(table.getDbName()).append('.').append(table.getTableName());
+    } else {
+      // In case of a non-partitioned table, the key for stats temporary store is "rootDir"
+      prefix.append(work.getAggKey());
+    }
+    if (partition != null) {
+      return Utilities.join(prefix.toString(), Warehouse.makePartPath(partition.getSpec()));
+    }
+    return prefix.toString();
+  }
+
+  private StatsAggregator createStatsAggregator(HiveConf conf) throws HiveException {
+    String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS);
+    StatsFactory factory = StatsFactory.newFactory(statsImpl, conf);
+    if (factory == null) {
+      throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg());
+    }
+    // initialize stats publishing table for noscan which has only stats task
+    // the rest of MR task following stats task initializes it in ExecDriver.java
+    StatsPublisher statsPublisher = factory.getStatsPublisher();
+    if (!statsPublisher.init(conf)) { // creating stats table if not exists
+      throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg());
+    }
+    Task sourceTask = getWork().getSourceTask();
+    if (sourceTask == null) {
+      throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg());
+    }
+    // manufacture a StatsAggregator
+    StatsAggregator statsAggregator = factory.getStatsAggregator();
+    if (!statsAggregator.connect(conf, sourceTask)) {
+      throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl));
+    }
+    return statsAggregator;
+  }
+
   private boolean existStats(Map<String, String> parameters) {
     return parameters.containsKey(StatsSetupConst.ROW_COUNT)
         || parameters.containsKey(StatsSetupConst.NUM_FILES)
@@ -374,31 +295,64 @@ public class StatsTask extends Task<Stat
         || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS);
   }
 
-  private void updateStats(String[] statsList, Statistics stats,
-      StatsAggregator statsAggregator, Map<String, String> parameters,
-      String aggKey, boolean atomic) throws HiveException {
-
-    String value;
-    Long longValue;
-    for (String statType : statsList) {
-      value = statsAggregator.aggregateStats(aggKey, statType);
-      if (value != null) {
-        longValue = Long.parseLong(value);
+  private void updateStats(StatsAggregator statsAggregator,
+      Map<String, String> parameters, String aggKey, boolean atomic) throws HiveException {
+
+    for (String statType : StatsSetupConst.statsRequireCompute) {
+      String value = statsAggregator.aggregateStats(aggKey, statType);
+      if (value != null && !value.isEmpty()) {
+        long longValue = Long.parseLong(value);
 
         if (work.getLoadTableDesc() != null &&
             !work.getLoadTableDesc().getReplace()) {
           String originalValue = parameters.get(statType);
           if (originalValue != null && !originalValue.equals("-1")) {
-            longValue += Long.parseLong(originalValue);
+            longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid
           }
         }
-        stats.setStat(statType, longValue);
+        parameters.put(statType, String.valueOf(longValue));
       } else {
         if (atomic) {
-          throw new HiveException("StatsAggregator failed to get statistics.");
+          throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType);
+        }
+      }
+    }
+  }
+
+  private void updateQuickStats(Warehouse wh, Map<String, String> parameters,
+      StorageDescriptor desc) throws MetaException {
+    /**
+     * calculate fast statistics
+     */
+    FileStatus[] partfileStatus = wh.getFileStatusesForSD(desc);
+    parameters.put(StatsSetupConst.NUM_FILES, String.valueOf(partfileStatus.length));
+    long partSize = 0L;
+    for (int i = 0; i < partfileStatus.length; i++) {
+      partSize += partfileStatus[i].getLen();
+    }
+    parameters.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(partSize));
+  }
+
+  private void clearStats(Map<String, String> parameters) {
+    for (String statType : StatsSetupConst.supportedStats) {
+      if (parameters.containsKey(statType)) {
+        parameters.put(statType, "0");
+      }
+    }
+  }
+
+  private String toString(Map<String, String> parameters) {
+    StringBuilder builder = new StringBuilder();
+    for (String statType : StatsSetupConst.supportedStats) {
+      String value = parameters.get(statType);
+      if (value != null) {
+        if (builder.length() > 0) {
+          builder.append(", ");
         }
+        builder.append(statType).append('=').append(value);
       }
     }
+    return builder.toString();
   }
 
   /**

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java Wed Dec 18 17:22:02 2013
@@ -29,8 +29,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
@@ -288,34 +286,21 @@ public class TableScanOperator extends O
       return;
     }
 
-    String key;
     String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(hconf));
     Map<String, String> statsToPublish = new HashMap<String, String>();
 
     for (String pspecs : stats.keySet()) {
       statsToPublish.clear();
-      if (pspecs.isEmpty()) {
-        if (statsPublisher instanceof CounterStatsPublisher) {
-          // key is of form : dbName.TblName/
-          key = conf.getStatsAggPrefix();
-       	 } else { 
-        // In case of a non-partitioned table, the key for temp storage is just
-        // "tableName + taskID"
-        String keyPrefix = Utilities.getHashedStatsPrefix(
-          conf.getStatsAggPrefix(), conf.getMaxStatsKeyPrefixLength());
-        key = keyPrefix + taskID;
-        } 
+      String prefix = Utilities.join(conf.getStatsAggPrefix(), pspecs);
+
+      String key;
+      int maxKeyLength = conf.getMaxStatsKeyPrefixLength();
+      if (statsPublisher instanceof CounterStatsPublisher) {
+        key = Utilities.getHashedStatsPrefix(prefix, maxKeyLength, 0);
       } else {
-         if (statsPublisher instanceof CounterStatsPublisher) {
-         // key is of form : dbName.tblName/p1=v1/ 
-         key = Utilities.appendPathSeparator(conf.getStatsAggPrefix()+pspecs);
-    	 } else {
-             // In case of a partition, the key for temp storage is
-             // "tableName + partitionSpecs + taskID"
-             String keyPrefix = Utilities.getHashedStatsPrefix(
-               conf.getStatsAggPrefix() + pspecs, conf.getMaxStatsKeyPrefixLength());
-               key = keyPrefix + taskID;
-        }
+        // stats publisher except counter type needs postfix 'taskID'
+        prefix = Utilities.getHashedStatsPrefix(prefix, maxKeyLength, taskID.length());
+        key = prefix + taskID;
       }
       for(String statType : stats.get(pspecs).getStoredStats()) {
         statsToPublish.put(statType, Long.toString(stats.get(pspecs).getStat(statType)));

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Wed Dec 18 17:22:02 2013
@@ -30,7 +30,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.DriverContext;
 import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.lib.Node;
@@ -89,7 +88,7 @@ public abstract class Task<T extends Ser
 
   public static enum FeedType {
     DYNAMIC_PARTITIONS, // list of dynamic partitions
-  };
+  }
 
   // Bean methods
 
@@ -168,12 +167,6 @@ public abstract class Task<T extends Ser
    */
   protected abstract int execute(DriverContext driverContext);
 
-  // dummy method - FetchTask overwrites this
-  public boolean fetch(ArrayList<String> res) throws IOException, CommandNeedRetryException {
-    assert false;
-    return false;
-  }
-
   public boolean isRootTask() {
     return rootTask;
   }
@@ -251,8 +244,6 @@ public abstract class Task<T extends Ser
         childTsk.removeFromChildrenTasks();
       }
     }
-
-    return;
   }
 
 

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Dec 18 17:22:02 2013
@@ -2345,25 +2345,35 @@ public final class Utilities {
    * @param maxPrefixLength
    * @return
    */
-  public static String getHashedStatsPrefix(String statsPrefix, int maxPrefixLength) {
-    String ret = appendPathSeparator(statsPrefix);
-    if (maxPrefixLength >= 0 && statsPrefix.length() > maxPrefixLength) {
+  public static String getHashedStatsPrefix(String statsPrefix,
+      int maxPrefixLength, int postfixLength) {
+    // todo: this might return possibly longer prefix than
+    // maxPrefixLength (if set) when maxPrefixLength - postfixLength < 17,
+    // which would make stat values invalid (especially for 'counter' type)
+    if (maxPrefixLength >= 0 && statsPrefix.length() > maxPrefixLength - postfixLength) {
       try {
         MessageDigest digester = MessageDigest.getInstance("MD5");
-        digester.update(ret.getBytes());
-        ret = new String(digester.digest()) + Path.SEPARATOR;
+        digester.update(statsPrefix.getBytes());
+        return new String(digester.digest()) + Path.SEPARATOR;  // 17 byte
       } catch (NoSuchAlgorithmException e) {
         throw new RuntimeException(e);
       }
     }
-    return ret;
+    return statsPrefix.endsWith(Path.SEPARATOR) ? statsPrefix : statsPrefix + Path.SEPARATOR;
   }
 
-  public static String appendPathSeparator(String path) {
-    if (!path.endsWith(Path.SEPARATOR)) {
-      path = path + Path.SEPARATOR;
+  public static String join(String... elements) {
+    StringBuilder builder = new StringBuilder();
+    for (String element : elements) {
+      if (element == null || element.isEmpty()) {
+        continue;
+      }
+      builder.append(element);
+      if (!element.endsWith(Path.SEPARATOR)) {
+        builder.append(Path.SEPARATOR);
+      }
     }
-    return path;
+    return builder.toString();
   }
 
   public static void setColumnNameList(JobConf jobConf, Operator op) {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java Wed Dec 18 17:22:02 2013
@@ -120,13 +120,20 @@ public class MathExpr {
 
     return current;
   }
-
+  
   // Convert all NaN values in vector v to NULL. Should only be used if n > 0.
   public static void NaNToNull(DoubleColumnVector v, int[] sel, boolean selectedInUse, int n) {
+    NaNToNull(v, sel, selectedInUse, n, false);
+  }
 
+  // Convert all NaN, and optionally infinity values in vector v to NULL.
+  // Should only be used if n > 0.
+  public static void NaNToNull(
+      DoubleColumnVector v, int[] sel, boolean selectedInUse, int n, boolean convertInfinity) {
     // handle repeating case
     if (v.isRepeating) {
-      if (Double.isNaN(v.vector[0])){
+      if ((convertInfinity && Double.isInfinite(v.vector[0])) || Double.isNaN(v.vector[0])){
+        v.vector[0] = DoubleColumnVector.NULL_VALUE;
         v.isNull[0] = true;
         v.noNulls = false;
       }
@@ -137,11 +144,11 @@ public class MathExpr {
       if (selectedInUse) {
         for(int j = 0; j != n; j++) {
           int i = sel[j];
-          if (Double.isNaN(v.vector[i])) {
+          if ((convertInfinity && Double.isInfinite(v.vector[i])) || Double.isNaN(v.vector[i])) {
+            v.vector[i] = DoubleColumnVector.NULL_VALUE;
             v.isNull[i] = true;
             v.noNulls = false;
           } else {
-
             // Must set isNull[i] to false to make sure
             // it gets initialized, in case we set noNulls to true.
             v.isNull[i] = false;
@@ -149,7 +156,8 @@ public class MathExpr {
         }
       } else {
         for(int i = 0; i != n; i++) {
-          if (Double.isNaN(v.vector[i])) {
+          if ((convertInfinity && Double.isInfinite(v.vector[i])) || Double.isNaN(v.vector[i])) {
+            v.vector[i] = DoubleColumnVector.NULL_VALUE;
             v.isNull[i] = true;
             v.noNulls = false;
           } else {
@@ -161,13 +169,15 @@ public class MathExpr {
       if (selectedInUse) {
         for (int j = 0; j != n; j++) {
           int i = sel[j];
-          if(Double.isNaN(v.vector[i])) {
+          if((convertInfinity && Double.isInfinite(v.vector[i])) || Double.isNaN(v.vector[i])) {
+            v.vector[i] = DoubleColumnVector.NULL_VALUE;
             v.isNull[i] = true;
           }
         }
       } else {
         for (int i = 0; i != n; i++) {
-          if(Double.isNaN(v.vector[i])) {
+          if((convertInfinity && Double.isInfinite(v.vector[i])) || Double.isNaN(v.vector[i])) {
+            v.vector[i] = DoubleColumnVector.NULL_VALUE;
             v.isNull[i] = true;
           }
         }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanMapper.java Wed Dec 18 17:22:02 2013
@@ -26,13 +26,13 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.io.RCFile.KeyBuffer;
 import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileKeyBufferWrapper;
 import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileValueBufferWrapper;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.stats.StatsFactory;
 import org.apache.hadoop.hive.ql.stats.StatsPublisher;
 import org.apache.hadoop.hive.shims.CombineHiveKey;
 import org.apache.hadoop.mapred.JobConf;
@@ -142,7 +142,7 @@ public class PartialScanMapper extends M
     // construct key used to store stats in intermediate db
     String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(jc));
     String keyPrefix = Utilities.getHashedStatsPrefix(
-        statsAggKeyPrefix, HiveConf.getIntVar(jc, ConfVars.HIVE_STATS_KEY_PREFIX_MAX_LENGTH));
+        statsAggKeyPrefix, StatsFactory.getMaxPrefixLength(jc), taskID.length());
     String key = keyPrefix + taskID;
 
     // construct statistics to be stored

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java Wed Dec 18 17:22:02 2013
@@ -75,7 +75,7 @@ public class GenMRTableScan1 implements 
 
     // create a dummy MapReduce task
     MapredWork currWork = GenMapRedUtils.getMapRedWork(parseCtx);
-    Task<? extends Serializable> currTask = TaskFactory.get(currWork, parseCtx.getConf());
+    MapRedTask currTask = (MapRedTask) TaskFactory.get(currWork, parseCtx.getConf());
     Operator<? extends OperatorDesc> currTopOp = op;
     ctx.setCurrTask(currTask);
     ctx.setCurrTopOp(currTopOp);
@@ -96,7 +96,7 @@ public class GenMRTableScan1 implements 
 
           StatsWork statsWork = new StatsWork(parseCtx.getQB().getParseInfo().getTableSpec());
           statsWork.setAggKey(op.getConf().getStatsAggPrefix());
-          statsWork.setSourceTask((MapRedTask)currTask);
+          statsWork.setSourceTask(currTask);
           statsWork.setStatsReliable(
             parseCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
           Task<StatsWork> statsTask = TaskFactory.get(statsWork, parseCtx.getConf());

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Wed Dec 18 17:22:02 2013
@@ -90,6 +90,7 @@ import org.apache.hadoop.hive.ql.plan.St
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.plan.TableScanDesc;
 import org.apache.hadoop.hive.ql.plan.TezWork;
+import org.apache.hadoop.hive.ql.stats.StatsFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.mapred.InputFormat;
 
@@ -1399,8 +1400,7 @@ public final class GenMapRedUtils {
     // mark the MapredWork and FileSinkOperator for gathering stats
     nd.getConf().setGatherStats(true);
     nd.getConf().setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE));
-    nd.getConf().setMaxStatsKeyPrefixLength(
-        hconf.getIntVar(ConfVars.HIVE_STATS_KEY_PREFIX_MAX_LENGTH));
+    nd.getConf().setMaxStatsKeyPrefixLength(StatsFactory.getMaxPrefixLength(hconf));
     // mrWork.addDestinationTable(nd.getConf().getTableInfo().getTableName());
 
     // subscribe feeds from the MoveTask so that MoveTask can forward the list

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Wed Dec 18 17:22:02 2013
@@ -143,6 +143,7 @@ public abstract class BaseSemanticAnalyz
     String collItemDelim = null;
     String mapKeyDelim = null;
     String lineDelim = null;
+    String nullFormat = null;
 
     protected void analyzeRowFormat(AnalyzeCreateCommonVars shared, ASTNode child) throws SemanticException {
       child = (ASTNode) child.getChild(0);
@@ -175,6 +176,10 @@ public abstract class BaseSemanticAnalyz
                 ErrorMsg.LINES_TERMINATED_BY_NON_NEWLINE.getMsg()));
           }
           break;
+        case HiveParser.TOK_TABLEROWFORMATNULL:
+          nullFormat = unescapeSQLString(rowChild.getChild(0)
+                    .getText());
+          break;
         default:
           assert false;
         }
@@ -750,7 +755,7 @@ public abstract class BaseSemanticAnalyz
         ASTNode partspec = (ASTNode) ast.getChild(1);
         partitions = new ArrayList<Partition>();
         // partSpec is a mapping from partition column name to its value.
-        partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
+        Map<String, String> tmpPartSpec = new HashMap<String, String>(partspec.getChildCount());
         for (int i = 0; i < partspec.getChildCount(); ++i) {
           ASTNode partspec_val = (ASTNode) partspec.getChild(i);
           String val = null;
@@ -765,15 +770,21 @@ public abstract class BaseSemanticAnalyz
           } else { // in the form of T partition (ds="2010-03-03")
             val = stripQuotes(partspec_val.getChild(1).getText());
           }
-          partSpec.put(colName, val);
+          tmpPartSpec.put(colName, val);
         }
 
         // check if the columns, as well as value types in the partition() clause are valid
-        validatePartSpec(tableHandle, partSpec, ast, conf);
+        validatePartSpec(tableHandle, tmpPartSpec, ast, conf);
+
+        List<FieldSchema> parts = tableHandle.getPartitionKeys();
+        partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
+        for (FieldSchema fs : parts) {
+          String partKey = fs.getName();
+          partSpec.put(partKey, tmpPartSpec.get(partKey));
+        }
 
         // check if the partition spec is valid
         if (numDynParts > 0) {
-          List<FieldSchema> parts = tableHandle.getPartitionKeys();
           int numStaPart = parts.size() - numDynParts;
           if (numStaPart == 0 &&
               conf.getVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equalsIgnoreCase("strict")) {
@@ -1211,8 +1222,8 @@ public abstract class BaseSemanticAnalyz
       Object convertedValue =
         ObjectInspectorConverters.getConverter(inputOI, outputOI).convert(value);
       if (convertedValue == null) {
-        throw new SemanticException(ErrorMsg.PARTITION_SPEC_TYPE_MISMATCH.format(astKeyName,
-            inputOI.getTypeName(), outputOI.getTypeName()));
+        throw new SemanticException(ErrorMsg.PARTITION_SPEC_TYPE_MISMATCH, astKeyName,
+            inputOI.getTypeName(), outputOI.getTypeName());
       }
 
       normalizeColSpec(partSpec, astKeyName, colType, colSpec, convertedValue);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Wed Dec 18 17:22:02 2013
@@ -1030,6 +1030,7 @@ public class DDLSemanticAnalyzer extends
             statDesc = new StatsWork(ltd);
           }
           statDesc.setNoStatsAggregator(true);
+          statDesc.setClearAggregatorStats(true);
           statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
           Task<? extends Serializable> statTask = TaskFactory.get(statDesc, conf);
           moveTsk.addDependentTask(statTask);
@@ -1643,6 +1644,7 @@ public class DDLSemanticAnalyzer extends
           statDesc = new StatsWork(ltd);
         }
         statDesc.setNoStatsAggregator(true);
+        statDesc.setClearAggregatorStats(true);
         statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
         Task<? extends Serializable> statTask = TaskFactory.get(statDesc, conf);
         moveTsk.addDependentTask(statTask);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Wed Dec 18 17:22:02 2013
@@ -176,6 +176,7 @@ KW_TBLPROPERTIES: 'TBLPROPERTIES';
 KW_IDXPROPERTIES: 'IDXPROPERTIES';
 KW_VALUE_TYPE: '$VALUE$';
 KW_ELEM_TYPE: '$ELEM$';
+KW_DEFINED: 'DEFINED';
 KW_CASE: 'CASE';
 KW_WHEN: 'WHEN';
 KW_THEN: 'THEN';

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Wed Dec 18 17:22:02 2013
@@ -180,6 +180,7 @@ TOK_TABLEROWFORMATFIELD;
 TOK_TABLEROWFORMATCOLLITEMS;
 TOK_TABLEROWFORMATMAPKEYS;
 TOK_TABLEROWFORMATLINES;
+TOK_TABLEROWFORMATNULL;
 TOK_TBLORCFILE;
 TOK_TBLSEQUENCEFILE;
 TOK_TBLTEXTFILE;
@@ -442,6 +443,7 @@ import java.util.HashMap;
     xlateMap.put("KW_PROPERTIES", "TBLPROPERTIES");
     xlateMap.put("KW_VALUE_TYPE", "\$VALUE\$");
     xlateMap.put("KW_ELEM_TYPE", "\$ELEM\$");
+    xlateMap.put("KW_DEFINED", "DEFINED");
 
     // Operators
     xlateMap.put("DOT", ".");
@@ -1538,8 +1540,8 @@ rowFormatDelimited
 @init { msgs.push("serde properties specification"); }
 @after { msgs.pop(); }
     :
-      KW_ROW KW_FORMAT KW_DELIMITED tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?
-    -> ^(TOK_SERDEPROPS tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?)
+      KW_ROW KW_FORMAT KW_DELIMITED tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier? tableRowNullFormat?
+    -> ^(TOK_SERDEPROPS tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier? tableRowNullFormat?)
     ;
 
 tableRowFormat
@@ -1621,6 +1623,13 @@ tableRowFormatLinesIdentifier
     -> ^(TOK_TABLEROWFORMATLINES $linesIdnt)
     ;
 
+tableRowNullFormat
+@init { msgs.push("table row format's null specifier"); }
+@after { msgs.pop(); }
+    :
+      KW_NULL KW_DEFINED KW_AS nullIdnt=StringLiteral
+    -> ^(TOK_TABLEROWFORMATNULL $nullIdnt)
+    ;
 tableFileFormat
 @init { msgs.push("table file format specification"); }
 @after { msgs.pop(); }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Wed Dec 18 17:22:02 2013
@@ -535,5 +535,5 @@ identifier
     
 nonReserved
     :
-    KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | 
 KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW
 _WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_INNER
+    KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | 
 KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW
 _WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_INNER | KW_DEFINED
     ;

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Dec 18 17:22:02 2013
@@ -155,6 +155,7 @@ import org.apache.hadoop.hive.ql.plan.pt
 import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
+import org.apache.hadoop.hive.ql.stats.StatsFactory;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
@@ -1297,6 +1298,7 @@ public class SemanticAnalyzer extends Ba
                   localDirectoryDesc.setCollItemDelim(rowFormatParams.collItemDelim);
                   localDirectoryDesc.setMapKeyDelim(rowFormatParams.mapKeyDelim);
                   localDirectoryDesc.setFieldEscape(rowFormatParams.fieldEscape);
+                  localDirectoryDesc.setNullFormat(rowFormatParams.nullFormat);
                   localDirectoryDescIsSet=true;
                   break;
                 case HiveParser.TOK_TABLESERIALIZER:
@@ -2304,6 +2306,10 @@ public class SemanticAnalyzer extends Ba
             throw new SemanticException(generateErrorMessage(rowChild,
                 ErrorMsg.LINES_TERMINATED_BY_NON_NEWLINE.getMsg()));
           }
+        case HiveParser.TOK_TABLEROWFORMATNULL:
+          String nullFormat = unescapeSQLString(rowChild.getChild(0).getText());
+          tblDesc.getProperties().setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT,
+              nullFormat);
           break;
         default:
           assert false;
@@ -8515,8 +8521,7 @@ public class SemanticAnalyzer extends Ba
     } else {
       tsDesc.setGatherStats(true);
       tsDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
-      tsDesc.setMaxStatsKeyPrefixLength(
-          conf.getIntVar(HiveConf.ConfVars.HIVE_STATS_KEY_PREFIX_MAX_LENGTH));
+      tsDesc.setMaxStatsKeyPrefixLength(StatsFactory.getMaxPrefixLength(conf));
 
       // append additional virtual columns for storing statistics
       Iterator<VirtualColumn> vcs = VirtualColumn.getStatsRegistry(conf).iterator();
@@ -9572,6 +9577,7 @@ public class SemanticAnalyzer extends Ba
           storageFormat.storageHandler, shared.serdeProps, tblProps, ifNotExists, skewedColNames,
           skewedValues);
       crtTblDesc.setStoredAsSubDirectories(storedAsDirs);
+      crtTblDesc.setNullFormat(rowFormatParams.nullFormat);
 
       crtTblDesc.validate();
       // outputs is empty, which means this create table happens in the current
@@ -9620,6 +9626,7 @@ public class SemanticAnalyzer extends Ba
           shared.serdeProps,
           tblProps, ifNotExists, skewedColNames, skewedValues);
       crtTblDesc.setStoredAsSubDirectories(storedAsDirs);
+      crtTblDesc.setNullFormat(rowFormatParams.nullFormat);
       qb.setTableDesc(crtTblDesc);
 
       SessionState.get().setCommandType(HiveOperation.CREATETABLE_AS_SELECT);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticException.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticException.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticException.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticException.java Wed Dec 18 17:22:02 2013
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.parse;
 
+import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 
 /**
@@ -44,4 +45,7 @@ public class SemanticException extends H
     super(message, cause);
   }
 
+  public SemanticException(ErrorMsg errorMsg, String... msgArgs) {
+    super(errorMsg, msgArgs);
+  }
 }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java Wed Dec 18 17:22:02 2013
@@ -62,6 +62,7 @@ public class CreateTableDesc extends DDL
   String collItemDelim;
   String mapKeyDelim;
   String lineDelim;
+  String nullFormat;
   String comment;
   String inputFormat;
   String outputFormat;
@@ -508,4 +509,20 @@ public class CreateTableDesc extends DDL
   public void setStoredAsSubDirectories(boolean isStoredAsSubDirectories) {
     this.isStoredAsSubDirectories = isStoredAsSubDirectories;
   }
+
+  /**
+   * @return the nullFormat
+   */
+  public String getNullFormat() {
+    return nullFormat;
+  }
+
+  /**
+   * Set null format string
+   * @param nullFormat
+   */
+  public void setNullFormat(String nullFormat) {
+    this.nullFormat = nullFormat;
+  }
+
 }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Wed Dec 18 17:22:02 2013
@@ -135,10 +135,15 @@ public final class PlanUtils {
       if (localDirectoryDesc.getSerName() != null) {
         tableDesc.getProperties().setProperty(
             serdeConstants.SERIALIZATION_LIB, localDirectoryDesc.getSerName());
-        }
+      }
       if (localDirectoryDesc.getOutputFormat() != null){
           tableDesc.setOutputFileFormatClass(Class.forName(localDirectoryDesc.getOutputFormat()));
       }
+      if (localDirectoryDesc.getNullFormat() != null) {
+        tableDesc.getProperties().setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT,
+              localDirectoryDesc.getNullFormat());
+      }
+
     } catch (ClassNotFoundException e) {
       // mimicking behaviour in CreateTableDesc tableDesc creation
       // returning null table description for output.
@@ -325,6 +330,11 @@ public final class PlanUtils {
         properties.setProperty(serdeConstants.LINE_DELIM, crtTblDesc.getLineDelim());
       }
 
+      if (crtTblDesc.getNullFormat() != null) {
+        properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT,
+              crtTblDesc.getNullFormat());
+      }
+
       if (crtTblDesc.getTableName() != null && crtTblDesc.getDatabaseName() != null) {
         properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME,
             crtTblDesc.getDatabaseName() + "." + crtTblDesc.getTableName());

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java Wed Dec 18 17:22:02 2013
@@ -50,7 +50,7 @@ public class StatsWork implements Serial
 
   private boolean isPartialScanAnalyzeCommand = false;
 
-  private transient Task<?> sourceTask;
+  private transient Task sourceTask;
 
   public StatsWork() {
   }
@@ -144,11 +144,11 @@ public class StatsWork implements Serial
     this.isPartialScanAnalyzeCommand = isPartialScanAnalyzeCommand;
   }
 
-  public Task<?> getSourceTask() {
+  public Task getSourceTask() {
     return sourceTask;
   }
 
-  public void setSourceTask(Task<?> sourceTask) {
+  public void setSourceTask(Task sourceTask) {
     this.sourceTask = sourceTask;
   }
 }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregator.java Wed Dec 18 17:22:02 2013
@@ -19,16 +19,15 @@
 package org.apache.hadoop.hive.ql.stats;
 
 import java.io.IOException;
-import java.util.Iterator;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.mr.ExecDriver;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
 import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.Counters.Counter;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RunningJob;
@@ -41,7 +40,7 @@ public class CounterStatsAggregator impl
   private JobClient jc;
 
   @Override
-  public boolean connect(Configuration hconf, Task<?> sourceTask) {
+  public boolean connect(Configuration hconf, Task sourceTask) {
     try {
       jc = new JobClient(toJobConf(hconf));
       RunningJob job = jc.getJob(((MapRedTask)sourceTask).getJobID());

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregatorTez.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregatorTez.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregatorTez.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsAggregatorTez.java Wed Dec 18 17:22:02 2013
@@ -40,7 +40,7 @@ public class CounterStatsAggregatorTez i
   }
 
   @Override
-  public boolean connect(Configuration hconf, Task<?> sourceTask) {
+  public boolean connect(Configuration hconf, Task sourceTask) {
     if (!(sourceTask instanceof TezTask)) {
       delegate = true;
       return mrAggregator.connect(hconf, sourceTask);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsAggregator.java Wed Dec 18 17:22:02 2013
@@ -35,7 +35,7 @@ public interface StatsAggregator {
    * @param sourceTask
    * @return true if connection is successful, false otherwise.
    */
-  public boolean connect(Configuration hconf, Task<?> sourceTask);
+  public boolean connect(Configuration hconf, Task sourceTask);
 
   /**
    * This method aggregates a given statistic from all tasks (partial stats).

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java Wed Dec 18 17:22:02 2013
@@ -28,6 +28,9 @@ import org.apache.hadoop.hive.common.Sta
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.util.ReflectionUtils;
 
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_STATS_KEY_PREFIX_MAX_LENGTH;
+
 /**
  * A factory of stats publisher and aggregator implementations of the
  * StatsPublisher and StatsAggregator interfaces.
@@ -40,8 +43,19 @@ public final class StatsFactory {
   private Class <? extends Serializable> aggregatorImplementation;
   private Configuration jobConf;
 
+  public static int getMaxPrefixLength(Configuration conf) {
+    int maxPrefixLength = HiveConf.getIntVar(conf, HIVE_STATS_KEY_PREFIX_MAX_LENGTH);
+    if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.counter.name())) {
+      // see org.apache.hadoop.mapred.Counter or org.apache.hadoop.mapreduce.MRJobConfig
+      int groupNameMax = conf.getInt("mapreduce.job.counters.group.name.max", 128);
+      maxPrefixLength = maxPrefixLength < 0 ? groupNameMax :
+          Math.min(maxPrefixLength, groupNameMax);
+    }
+    return maxPrefixLength;
+  }
+
   public static StatsFactory newFactory(Configuration conf) {
-    return newFactory(HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS), conf);
+    return newFactory(HiveConf.getVar(conf, HIVESTATSDBCLASS), conf);
   }
 
   /**

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Wed Dec 18 17:22:02 2013
@@ -55,7 +55,7 @@ public class JDBCStatsAggregator impleme
   }
 
   @Override
-  public boolean connect(Configuration hiveconf, Task<?> sourceTask) {
+  public boolean connect(Configuration hiveconf, Task sourceTask) {
     this.hiveconf = hiveconf;
     timeout = HiveConf.getIntVar(hiveconf, HiveConf.ConfVars.HIVE_STATS_JDBC_TIMEOUT);
     connectionString = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSDBCONNECTIONSTRING);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Wed Dec 18 17:22:02 2013
@@ -111,6 +111,7 @@ public class JDBCStatsPublisher implemen
       } catch (SQLException e) {
         // for SQLTransientException (maxRetries already achieved at Utilities retry functions
         // or SQLNonTransientException, declare a real failure
+        LOG.error("Error during JDBC connection to " + connectionString + ". ", e);
         return false;
       }
     }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java Wed Dec 18 17:22:02 2013
@@ -35,8 +35,6 @@ import org.apache.hadoop.hive.serde2.io.
     + "  > SELECT _FUNC_(10) FROM src LIMIT 1;\n" + "  1")
 @VectorizedExpressions({FuncLog10LongToDouble.class, FuncLog10DoubleToDouble.class})
 public class UDFLog10 extends UDF {
-  private static double log10 = Math.log(10.0);
-
   private final DoubleWritable result = new DoubleWritable();
 
   public UDFLog10() {
@@ -49,9 +47,8 @@ public class UDFLog10 extends UDF {
     if (a == null || a.get() <= 0.0) {
       return null;
     } else {
-      result.set(Math.log(a.get()) / log10);
+      result.set(Math.log10(a.get()));
       return result;
     }
   }
-
 }

Modified: hive/branches/tez/ql/src/test/queries/clientpositive/stats_counter.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/stats_counter.q?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/stats_counter.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/stats_counter.q Wed Dec 18 17:22:02 2013
@@ -1,6 +1,16 @@
-set hive.stats.autogather=true;
 set hive.stats.dbclass=counter;
+set hive.stats.autogather=false;
+
+-- by analyze
+create table dummy1 as select * from src;
+
+analyze table dummy1 compute statistics;
+desc formatted dummy1;
+
+set hive.stats.dbclass=counter;
+set hive.stats.autogather=true;
 
-create table dummy as select * from src;
+-- by autogather
+create table dummy2 as select * from src;
 
-desc formatted dummy;
+desc formatted dummy2;

Modified: hive/branches/tez/ql/src/test/queries/clientpositive/stats_noscan_2.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/stats_noscan_2.q?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/stats_noscan_2.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/stats_noscan_2.q Wed Dec 18 17:22:02 2013
@@ -3,10 +3,10 @@
 -- 1 test table
 CREATE EXTERNAL TABLE anaylyze_external (a INT) LOCATION '${system:hive.root}/data/files/ext_test';
 SELECT * FROM anaylyze_external;
-analyze table anaylyze_external compute statistics;
-describe formatted anaylyze_external;
 analyze table anaylyze_external compute statistics noscan;
 describe formatted anaylyze_external;
+analyze table anaylyze_external compute statistics;
+describe formatted anaylyze_external;
 drop table anaylyze_external;
 
 -- 2 test partition
@@ -21,10 +21,10 @@ CREATE EXTERNAL TABLE anaylyze_external 
 ALTER TABLE anaylyze_external ADD PARTITION (insertdate='2008-01-01') location 'pfile://${system:test.tmp.dir}/texternal/2008-01-01';
 select count(*) from anaylyze_external where insertdate='2008-01-01';
 -- analyze
-analyze table anaylyze_external PARTITION (insertdate='2008-01-01') compute statistics;
-describe formatted anaylyze_external PARTITION (insertdate='2008-01-01');
 analyze table anaylyze_external PARTITION (insertdate='2008-01-01') compute statistics noscan;
 describe formatted anaylyze_external PARTITION (insertdate='2008-01-01');
+analyze table anaylyze_external PARTITION (insertdate='2008-01-01') compute statistics;
+describe formatted anaylyze_external PARTITION (insertdate='2008-01-01');
 dfs -rmr ${system:test.tmp.dir}/texternal;
 drop table anaylyze_external;
 

Modified: hive/branches/tez/ql/src/test/queries/clientpositive/vectorized_math_funcs.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/vectorized_math_funcs.q?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/vectorized_math_funcs.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/vectorized_math_funcs.q Wed Dec 18 17:22:02 2013
@@ -16,6 +16,8 @@ select
   ,Log10(cdouble)
   -- Use log2 as a representative function to test all input types.
   ,Log2(cdouble)
+  -- Use 15601.0 to test zero handling, as there are no zeroes in the table
+  ,Log2(cdouble - 15601.0)
   ,Log2(cfloat)
   ,Log2(cbigint)
   ,Log2(cint)
@@ -66,6 +68,8 @@ select
   ,Log10(cdouble)
   -- Use log2 as a representative function to test all input types.
   ,Log2(cdouble)
+  -- Use 15601.0 to test zero handling, as there are no zeroes in the table
+  ,Log2(cdouble - 15601.0)
   ,Log2(cfloat)
   ,Log2(cbigint)
   ,Log2(cint)

Modified: hive/branches/tez/ql/src/test/results/clientnegative/dyn_part4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientnegative/dyn_part4.q.out?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientnegative/dyn_part4.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientnegative/dyn_part4.q.out Wed Dec 18 17:22:02 2013
@@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE
 POSTHOOK: query: create table nzhang_part4 (key string) partitioned by (ds string, hr string, value string)
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: default@nzhang_part4
-FAILED: SemanticException [Error 10125]: Partition columns in partition specification are not the same as that defined in the table schema. The names and orders have to be exactly the same. Partition columns in the table schema are: (ds, hr, value), while the partitions specified in the query are: (value, ds, hr).
+FAILED: SemanticException [Error 10094]: Line 3:46 Dynamic partition cannot be the parent of a static partition 'hr'

Modified: hive/branches/tez/ql/src/test/results/clientnegative/stats_aggregator_error_1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientnegative/stats_aggregator_error_1.q.out?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientnegative/stats_aggregator_error_1.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientnegative/stats_aggregator_error_1.q.out Wed Dec 18 17:22:02 2013
@@ -19,6 +19,7 @@ PREHOOK: query: INSERT OVERWRITE TABLE t
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
 PREHOOK: Output: default@tmptable
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type custom cannot be connected to
 POSTHOOK: query: INSERT OVERWRITE TABLE tmptable select * from src
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src

Modified: hive/branches/tez/ql/src/test/results/clientnegative/stats_aggregator_error_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientnegative/stats_aggregator_error_2.q.out?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientnegative/stats_aggregator_error_2.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientnegative/stats_aggregator_error_2.q.out Wed Dec 18 17:22:02 2013
@@ -17,6 +17,7 @@ PREHOOK: query: INSERT OVERWRITE TABLE t
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
 PREHOOK: Output: default@tmptable
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30000]: StatsPublisher cannot be obtained. There was a error to retrieve the StatsPublisher, and retrying might help. If you dont want the query to fail because accurate statistics could not be collected, set hive.stats.reliable=false
 POSTHOOK: query: INSERT OVERWRITE TABLE tmptable select * from src
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src

Modified: hive/branches/tez/ql/src/test/results/clientnegative/stats_publisher_error_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientnegative/stats_publisher_error_2.q.out?rev=1552041&r1=1552040&r2=1552041&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientnegative/stats_publisher_error_2.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientnegative/stats_publisher_error_2.q.out Wed Dec 18 17:22:02 2013
@@ -17,6 +17,7 @@ PREHOOK: query: INSERT OVERWRITE TABLE t
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
 PREHOOK: Output: default@tmptable
+[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30000]: StatsPublisher cannot be obtained. There was a error to retrieve the StatsPublisher, and retrying might help. If you dont want the query to fail because accurate statistics could not be collected, set hive.stats.reliable=false
 POSTHOOK: query: INSERT OVERWRITE TABLE tmptable select * from src
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src



Mime
View raw message