hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1475662 [1/2] - in /hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/io/ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/ ql/src/java/org/apache/h...
Date Thu, 25 Apr 2013 08:03:49 GMT
Author: namit
Date: Thu Apr 25 08:03:48 2013
New Revision: 1475662

URL: http://svn.apache.org/r1475662
Log:
HIVE-4005 Column truncation
(Kevin Wilfong via namit)


Added:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java
    hive/trunk/ql/src/test/queries/clientnegative/truncate_bucketed_column.q
    hive/trunk/ql/src/test/queries/clientnegative/truncate_column_indexed_table.q
    hive/trunk/ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q
    hive/trunk/ql/src/test/queries/clientnegative/truncate_column_seqfile.q
    hive/trunk/ql/src/test/queries/clientnegative/truncate_nonexistant_column.q
    hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column.q
    hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column2.q
    hive/trunk/ql/src/test/queries/clientpositive/truncate_column.q
    hive/trunk/ql/src/test/queries/clientpositive/truncate_column_buckets.q
    hive/trunk/ql/src/test/queries/clientpositive/truncate_column_list_bucket.q
    hive/trunk/ql/src/test/queries/clientpositive/truncate_column_merge.q
    hive/trunk/ql/src/test/results/clientnegative/truncate_bucketed_column.q.out
    hive/trunk/ql/src/test/results/clientnegative/truncate_column_archived.q.out
    hive/trunk/ql/src/test/results/clientnegative/truncate_column_indexed_table.q.out
    hive/trunk/ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out
    hive/trunk/ql/src/test/results/clientnegative/truncate_column_seqfile.q.out
    hive/trunk/ql/src/test/results/clientnegative/truncate_nonexistant_column.q.out
    hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out
    hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column2.q.out
    hive/trunk/ql/src/test/results/clientpositive/truncate_column.q.out
    hive/trunk/ql/src/test/results/clientpositive/truncate_column_buckets.q.out
    hive/trunk/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out
    hive/trunk/ql/src/test/results/clientpositive/truncate_column_merge.q.out
Modified:
    hive/trunk/build-common.xml
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileValueBufferWrapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java

Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Thu Apr 25 08:03:48 2013
@@ -59,7 +59,7 @@
   <property name="test.output" value="true"/>
   <property name="test.junit.output.format" value="xml"/>
   <property name="test.junit.output.usefile" value="true"/>
-  <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q"/>
+  <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q,truncate_column_buckets.q"/>
   <property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
   <property name="test.silent" value="true"/>
   <property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Thu Apr 25 08:03:48 2013
@@ -321,6 +321,13 @@ public enum ErrorMsg {
     "with distincts. Either set hive.new.job.grouping.set.cardinality to a high number " +
     "(higher than the number of rows per input row due to grouping sets in the query), or " +
     "rewrite the query to not use distincts."),
+  TRUNCATE_COLUMN_INDEXED_TABLE(10227, "Can not truncate columns from table with indexes"),
+  TRUNCATE_COLUMN_NOT_RC(10228, "Only RCFileFormat supports column truncation."),
+  TRUNCATE_COLUMN_ARCHIVED(10229, "Column truncation cannot be performed on archived partitions."),
+  TRUNCATE_BUCKETED_COLUMN(10230,
+      "A column on which a partition/table is bucketed cannot be truncated."),
+  TRUNCATE_LIST_BUCKETED_COLUMN(10231,
+      "A column on which a partition/table is list bucketed cannot be truncated."),
 
   OPERATOR_NOT_ALLOWED_WITH_MAPJOIN(10227,
     "Not all clauses are supported with mapjoin hint. Please remove mapjoin hint."),

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Thu Apr 25 08:03:48 2013
@@ -40,10 +40,10 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.Map.Entry;
 
 import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.commons.lang.StringUtils;
@@ -86,6 +86,8 @@ import org.apache.hadoop.hive.ql.hooks.R
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask;
 import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork;
+import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateTask;
+import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateWork;
 import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
 import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
 import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
@@ -111,7 +113,6 @@ import org.apache.hadoop.hive.ql.plan.Al
 import org.apache.hadoop.hive.ql.plan.AlterIndexDesc;
 import org.apache.hadoop.hive.ql.plan.AlterTableAlterPartDesc;
 import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
 import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc;
 import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc;
 import org.apache.hadoop.hive.ql.plan.CreateIndexDesc;
@@ -150,6 +151,7 @@ import org.apache.hadoop.hive.ql.plan.Sh
 import org.apache.hadoop.hive.ql.plan.SwitchDatabaseDesc;
 import org.apache.hadoop.hive.ql.plan.TruncateTableDesc;
 import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
 import org.apache.hadoop.hive.ql.security.authorization.Privilege;
 import org.apache.hadoop.hive.serde.serdeConstants;
@@ -3951,6 +3953,21 @@ public class DDLTask extends Task<DDLWor
   }
 
   private int truncateTable(Hive db, TruncateTableDesc truncateTableDesc) throws HiveException {
+
+    if (truncateTableDesc.getColumnIndexes() != null) {
+      ColumnTruncateWork truncateWork = new ColumnTruncateWork(
+          truncateTableDesc.getColumnIndexes(), truncateTableDesc.getInputDir(),
+          truncateTableDesc.getOutputDir());
+      truncateWork.setListBucketingCtx(truncateTableDesc.getLbCtx());
+      truncateWork.setMapperCannotSpanPartns(true);
+      DriverContext driverCxt = new DriverContext();
+      ColumnTruncateTask taskExec = new ColumnTruncateTask();
+      taskExec.initialize(db.getConf(), null, driverCxt);
+      taskExec.setWork(truncateWork);
+      taskExec.setQueryPlan(this.getQueryPlan());
+      return taskExec.execute(driverCxt);
+    }
+
     String tableName = truncateTableDesc.getTableName();
     Map<String, String> partSpec = truncateTableDesc.getPartSpec();
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java Thu Apr 25 08:03:48 2013
@@ -243,6 +243,12 @@ public class RCFile {
       this.numberRows = numberRows;
     }
 
+    public void nullColumn(int columnIndex) {
+      eachColumnValueLen[columnIndex] = 0;
+      eachColumnUncompressedValueLen[columnIndex] = 0;
+      allCellValLenBuffer[columnIndex] = new NonSyncDataOutputBuffer();
+    }
+
     /**
      * add in a new column's meta data.
      *
@@ -553,6 +559,14 @@ public class RCFile {
       }
     }
 
+    public void nullColumn(int columnIndex) {
+      if (codec != null) {
+        compressedColumnsValueBuffer[columnIndex].reset();
+      } else {
+        loadedColumnsValueBuffer[columnIndex].reset();
+      }
+    }
+
     public void clearColumnBuffer() throws IOException {
       decompressBuffer.reset();
     }
@@ -1077,6 +1091,7 @@ public class RCFile {
       public int rowReadIndex;
       public int runLength;
       public int prvLength;
+      public boolean isNulled;
     }
     private final Path file;
     private final FSDataInputStream in;
@@ -1491,6 +1506,7 @@ public class RCFile {
         col.rowReadIndex = 0;
         col.runLength = 0;
         col.prvLength = -1;
+        col.isNulled = colValLenBufferReadIn[selIx].getLength() == 0;
       }
 
       return currentKeyLength;
@@ -1694,18 +1710,22 @@ public class RCFile {
           SelectedColumn col = selectedColumns[j];
           int i = col.colIndex;
 
-          BytesRefWritable ref = ret.unCheckedGet(i);
+          if (col.isNulled) {
+            ret.set(i, null);
+          } else {
+            BytesRefWritable ref = ret.unCheckedGet(i);
 
-          colAdvanceRow(j, col);
+            colAdvanceRow(j, col);
 
-          if (currentValue.decompressedFlag[j]) {
-            ref.set(currentValue.loadedColumnsValueBuffer[j].getData(),
-                col.rowReadIndex, col.prvLength);
-          } else {
-            ref.set(currentValue.lazyDecompressCallbackObjs[j],
-                col.rowReadIndex, col.prvLength);
+            if (currentValue.decompressedFlag[j]) {
+              ref.set(currentValue.loadedColumnsValueBuffer[j].getData(),
+                  col.rowReadIndex, col.prvLength);
+            } else {
+              ref.set(currentValue.lazyDecompressCallbackObjs[j],
+                  col.rowReadIndex, col.prvLength);
+            }
+            col.rowReadIndex += col.prvLength;
           }
-          col.rowReadIndex += col.prvLength;
         }
       } else {
         // This version of the loop eliminates a condition check and branch
@@ -1714,12 +1734,16 @@ public class RCFile {
           SelectedColumn col = selectedColumns[j];
           int i = col.colIndex;
 
-          BytesRefWritable ref = ret.unCheckedGet(i);
+          if (col.isNulled) {
+            ret.set(i, null);
+          } else {
+            BytesRefWritable ref = ret.unCheckedGet(i);
 
-          colAdvanceRow(j, col);
-          ref.set(currentValue.loadedColumnsValueBuffer[j].getData(),
-                col.rowReadIndex, col.prvLength);
-          col.rowReadIndex += col.prvLength;
+            colAdvanceRow(j, col);
+            ref.set(currentValue.loadedColumnsValueBuffer[j].getData(),
+                  col.rowReadIndex, col.prvLength);
+            col.rowReadIndex += col.prvLength;
+          }
         }
       }
       rowFetched = true;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileKeyBufferWrapper.java Thu Apr 25 08:03:48 2013
@@ -38,7 +38,7 @@ public class RCFileKeyBufferWrapper impl
 
   protected CompressionCodec codec;
 
-  protected RCFileKeyBufferWrapper() {
+  public RCFileKeyBufferWrapper() {
   }
 
   public static RCFileKeyBufferWrapper create(KeyBuffer currentKeyBufferObj) {
@@ -66,4 +66,48 @@ public class RCFileKeyBufferWrapper impl
     return keyBuffer;
   }
 
+  public void setKeyBuffer(KeyBuffer keyBuffer) {
+    this.keyBuffer = keyBuffer;
+  }
+
+  public int getRecordLength() {
+    return recordLength;
+  }
+
+  public void setRecordLength(int recordLength) {
+    this.recordLength = recordLength;
+  }
+
+  public int getKeyLength() {
+    return keyLength;
+  }
+
+  public void setKeyLength(int keyLength) {
+    this.keyLength = keyLength;
+  }
+
+  public int getCompressedKeyLength() {
+    return compressedKeyLength;
+  }
+
+  public void setCompressedKeyLength(int compressedKeyLength) {
+    this.compressedKeyLength = compressedKeyLength;
+  }
+
+  public Path getInputPath() {
+    return inputPath;
+  }
+
+  public void setInputPath(Path inputPath) {
+    this.inputPath = inputPath;
+  }
+
+  public CompressionCodec getCodec() {
+    return codec;
+  }
+
+  public void setCodec(CompressionCodec codec) {
+    this.codec = codec;
+  }
+
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileValueBufferWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileValueBufferWrapper.java?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileValueBufferWrapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileValueBufferWrapper.java Thu Apr 25 08:03:48 2013
@@ -48,4 +48,12 @@ public class RCFileValueBufferWrapper im
     return this.valueBuffer.compareTo(o.valueBuffer);
   }
 
+  public ValueBuffer getValueBuffer() {
+    return valueBuffer;
+  }
+
+  public void setValueBuffer(ValueBuffer valueBuffer) {
+    this.valueBuffer = valueBuffer;
+  }
+
 }

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateMapper.java Thu Apr 25 08:03:48 2013
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.rcfile.truncate;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.RCFile;
+import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileKeyBufferWrapper;
+import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileValueBufferWrapper;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.hive.shims.CombineHiveKey;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+
+@SuppressWarnings("deprecation")
+public class ColumnTruncateMapper extends MapReduceBase implements
+    Mapper<Object, RCFileValueBufferWrapper, Object, Object> {
+
+  private JobConf jc;
+  Class<? extends Writable> outputClass;
+  RCFile.Writer outWriter;
+
+  Path finalPath;
+  FileSystem fs;
+
+  boolean exception = false;
+  boolean autoDelete = false;
+  Path outPath;
+
+  CompressionCodec codec = null;
+  int columnNumber = 0;
+
+  boolean tmpPathFixedConcatenate = false;
+  boolean tmpPathFixed = false;
+  Path tmpPath;
+  Path taskTmpPath;
+  Path dpPath;
+  ColumnTruncateWork work;
+
+  public final static Log LOG = LogFactory.getLog(ColumnTruncateMapper.class.getName());
+
+  public ColumnTruncateMapper() {
+  }
+
+  @Override
+  public void configure(JobConf job) {
+    jc = job;
+    work = (ColumnTruncateWork) Utilities.getMapRedWork(job);
+
+    String specPath = work.getOutputDir();
+    Path tmpPath = Utilities.toTempPath(specPath);
+    Path taskTmpPath = Utilities.toTaskTempPath(specPath);
+    updatePaths(tmpPath, taskTmpPath);
+    try {
+      fs = (new Path(specPath)).getFileSystem(job);
+      autoDelete = ShimLoader.getHadoopShims().fileSystemDeleteOnExit(fs,
+          outPath);
+    } catch (IOException e) {
+      this.exception = true;
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void updatePaths(Path tmpPath, Path taskTmpPath) {
+    String taskId = Utilities.getTaskId(jc);
+    this.tmpPath = tmpPath;
+    this.taskTmpPath = taskTmpPath;
+    String inputFile = HiveConf.getVar(jc, HiveConf.ConfVars.HADOOPMAPFILENAME);
+    int lastSeparator = inputFile.lastIndexOf(Path.SEPARATOR) + 1;
+    finalPath = new Path(tmpPath, inputFile.substring(lastSeparator));
+    outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId));
+  }
+
+  @Override
+  public void map(Object k, RCFileValueBufferWrapper value,
+      OutputCollector<Object, Object> output, Reporter reporter)
+      throws IOException {
+    try {
+
+      RCFileKeyBufferWrapper key = null;
+      if (k instanceof CombineHiveKey) {
+        key = (RCFileKeyBufferWrapper) ((CombineHiveKey) k).getKey();
+      } else {
+        key = (RCFileKeyBufferWrapper) k;
+      }
+
+      if (work.getListBucketingCtx().calculateListBucketingLevel() > 0) {
+        if (!this.tmpPathFixedConcatenate) {
+          fixTmpPathConcatenate(key.getInputPath().getParent(),
+              work.getListBucketingCtx().calculateListBucketingLevel());
+          tmpPathFixedConcatenate = true;
+        }
+      }
+
+      if (outWriter == null) {
+        codec = key.getCodec();
+        columnNumber = key.getKeyBuffer().getColumnNumber();
+        jc.setInt(RCFile.COLUMN_NUMBER_CONF_STR, columnNumber);
+        outWriter = new RCFile.Writer(fs, jc, outPath, null, codec);
+      }
+
+      for (Integer i : work.getDroppedColumns()) {
+        key.getKeyBuffer().nullColumn(i);
+        value.getValueBuffer().nullColumn(i);
+      }
+
+      int keyLength = key.getKeyBuffer().getSize();
+      int recordLength = key.getKeyBuffer().getSize();
+      for (int columnLen : key.getKeyBuffer().getEachColumnValueLen()) {
+        recordLength += columnLen;
+      }
+
+      outWriter.flushBlock(key.getKeyBuffer(), value.getValueBuffer(), recordLength,
+          keyLength, key.getCompressedKeyLength());
+    } catch (Throwable e) {
+      this.exception = true;
+      close();
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Fixes tmpPath to point to the correct list bucketing sub-directories.
+   * Before this is called, tmpPath will default to the root tmp table dir
+   * Reason to add a new method instead of changing fixTmpPath()
+   * Reason 1: logic has slightly difference
+   * fixTmpPath(..) needs 2 variables in order to decide path delta which is in variable newPath.
+   * 1. inputPath.depth()
+   * 2. tmpPath.depth()
+   * fixTmpPathConcatenate needs 2 variables too but one of them is different from fixTmpPath(..)
+   * 1. inputPath.depth()
+   * 2. listBucketingDepth
+   * Reason 2: less risks
+   * The existing logic is a little not trivial around map() and fixTmpPath(). In order to ensure
+   * minimum impact on existing flow, we try to avoid change on existing code/flow but add new code
+   * for new feature.
+   *
+   * @param inputPath
+   * @throws HiveException
+   * @throws IOException
+   */
+  private void fixTmpPathConcatenate(Path inputPath, int listBucketingDepth)
+      throws HiveException, IOException {
+    dpPath = inputPath;
+    Path newPath = new Path(".");
+
+    int depth = listBucketingDepth;
+    // Build the path from bottom up. pick up list bucketing subdirectories
+    while ((inputPath != null) && (depth > 0)) {
+      newPath = new Path(inputPath.getName(), newPath);
+      inputPath = inputPath.getParent();
+      depth--;
+    }
+
+    Path newTmpPath = new Path(tmpPath, newPath);
+    Path newTaskTmpPath = new Path(taskTmpPath, newPath);
+    if (!fs.exists(newTmpPath)) {
+      fs.mkdirs(newTmpPath);
+    }
+    updatePaths(newTmpPath, newTaskTmpPath);
+  }
+
+
+  @Override
+  public void close() throws IOException {
+    // close writer
+    if (outWriter == null) {
+      return;
+    }
+
+    outWriter.close();
+    outWriter = null;
+
+    if (!exception) {
+      FileStatus fss = fs.getFileStatus(outPath);
+      LOG.info("renamed path " + outPath + " to " + finalPath
+          + " . File size is " + fss.getLen());
+      if (!fs.rename(outPath, finalPath)) {
+        throw new IOException("Unable to rename output to " + finalPath);
+      }
+    } else {
+      if (!autoDelete) {
+        fs.delete(outPath, true);
+      }
+    }
+  }
+
+  public static String BACKUP_PREFIX = "_backup.";
+
+  public static Path backupOutputPath(FileSystem fs, Path outpath, JobConf job)
+      throws IOException, HiveException {
+    if (fs.exists(outpath)) {
+      Path backupPath = new Path(outpath.getParent(), BACKUP_PREFIX
+          + outpath.getName());
+      Utilities.rename(fs, outpath, backupPath);
+      return backupPath;
+    } else {
+      return null;
+    }
+  }
+
+  public static void jobClose(String outputPath, boolean success, JobConf job,
+      LogHelper console, DynamicPartitionCtx dynPartCtx, Reporter reporter
+      ) throws HiveException, IOException {
+    Path outpath = new Path(outputPath);
+    FileSystem fs = outpath.getFileSystem(job);
+    Path backupPath = backupOutputPath(fs, outpath, job);
+    Utilities.mvFileToFinalPath(outputPath, job, success, LOG, dynPartCtx, null,
+      reporter);
+    fs.delete(backupPath, true);
+  }
+
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java Thu Apr 25 08:03:48 2013
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.rcfile.truncate;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.HadoopJobExecHelper;
+import org.apache.hadoop.hive.ql.exec.HadoopJobExecHook;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.Throttle;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
+import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
+
+@SuppressWarnings( { "deprecation", "unchecked" })
+public class ColumnTruncateTask extends Task<ColumnTruncateWork> implements Serializable,
+    HadoopJobExecHook {
+
+  private static final long serialVersionUID = 1L;
+
+  protected transient JobConf job;
+  protected HadoopJobExecHelper jobExecHelper;
+
+  @Override
+  public void initialize(HiveConf conf, QueryPlan queryPlan,
+      DriverContext driverContext) {
+    super.initialize(conf, queryPlan, driverContext);
+    job = new JobConf(conf, ColumnTruncateTask.class);
+    jobExecHelper = new HadoopJobExecHelper(job, this.console, this, this);
+  }
+
+  @Override
+  public boolean requireLock() {
+    return true;
+  }
+
+  boolean success = true;
+
+  @Override
+  /**
+   * start a new map-reduce job to do the truncation, almost the same as ExecDriver.
+   */
+  public int execute(DriverContext driverContext) {
+    HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT,
+        BucketizedHiveInputFormat.class.getName());
+    success = true;
+    ShimLoader.getHadoopShims().prepareJobOutput(job);
+    job.setOutputFormat(HiveOutputFormatImpl.class);
+    job.setMapperClass(work.getMapperClass());
+
+    Context ctx = driverContext.getCtx();
+    boolean ctxCreated = false;
+    try {
+      if (ctx == null) {
+        ctx = new Context(job);
+        ctxCreated = true;
+      }
+    }catch (IOException e) {
+      e.printStackTrace();
+      console.printError("Error launching map-reduce job", "\n"
+          + org.apache.hadoop.util.StringUtils.stringifyException(e));
+      return 5;
+    }
+
+    job.setMapOutputKeyClass(NullWritable.class);
+    job.setMapOutputValueClass(NullWritable.class);
+    if(work.getNumMapTasks() != null) {
+      job.setNumMapTasks(work.getNumMapTasks());
+    }
+
+    // zero reducers
+    job.setNumReduceTasks(0);
+
+    if (work.getMinSplitSize() != null) {
+      HiveConf.setLongVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE, work
+          .getMinSplitSize().longValue());
+    }
+
+    if (work.getInputformat() != null) {
+      HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, work
+          .getInputformat());
+    }
+
+    String inpFormat = HiveConf.getVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT);
+    if ((inpFormat == null) || (!StringUtils.isNotBlank(inpFormat))) {
+      inpFormat = ShimLoader.getHadoopShims().getInputFormatClassName();
+    }
+
+    LOG.info("Using " + inpFormat);
+
+    try {
+      job.setInputFormat((Class<? extends InputFormat>) (Class
+          .forName(inpFormat)));
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e.getMessage());
+    }
+
+    String outputPath = this.work.getOutputDir();
+    Path tempOutPath = Utilities.toTempPath(new Path(outputPath));
+    try {
+      FileSystem fs = tempOutPath.getFileSystem(job);
+      if (!fs.exists(tempOutPath)) {
+        fs.mkdirs(tempOutPath);
+      }
+    } catch (IOException e) {
+      console.printError("Can't make path " + outputPath + " : " + e.getMessage());
+      return 6;
+    }
+
+    job.setOutputKeyClass(NullWritable.class);
+    job.setOutputValueClass(NullWritable.class);
+
+    int returnVal = 0;
+    RunningJob rj = null;
+    boolean noName = StringUtils.isEmpty(HiveConf.getVar(job,
+        HiveConf.ConfVars.HADOOPJOBNAME));
+
+    String jobName = null;
+    if (noName && this.getQueryPlan() != null) {
+      int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
+      jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(),
+          maxlen - 6);
+    }
+
+    if (noName) {
+      // This is for a special case to ensure unit tests pass
+      HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME,
+          jobName != null ? jobName : "JOB" + Utilities.randGen.nextInt());
+    }
+
+    try {
+      addInputPaths(job, work);
+
+      Utilities.setMapRedWork(job, work, ctx.getMRTmpFileURI());
+
+      // remove the pwd from conf file so that job tracker doesn't show this
+      // logs
+      String pwd = HiveConf.getVar(job, HiveConf.ConfVars.METASTOREPWD);
+      if (pwd != null) {
+        HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE");
+      }
+      JobClient jc = new JobClient(job);
+
+      String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR);
+      if (!addedJars.isEmpty()) {
+        job.set("tmpjars", addedJars);
+      }
+
+      // make this client wait if job trcker is not behaving well.
+      Throttle.checkJobTracker(job, LOG);
+
+      // Finally SUBMIT the JOB!
+      rj = jc.submitJob(job);
+
+      returnVal = jobExecHelper.progress(rj, jc);
+      success = (returnVal == 0);
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      String mesg = " with exception '" + Utilities.getNameMessage(e) + "'";
+      if (rj != null) {
+        mesg = "Ended Job = " + rj.getJobID() + mesg;
+      } else {
+        mesg = "Job Submission failed" + mesg;
+      }
+
+      // Has to use full name to make sure it does not conflict with
+      // org.apache.commons.lang.StringUtils
+      console.printError(mesg, "\n"
+          + org.apache.hadoop.util.StringUtils.stringifyException(e));
+
+      success = false;
+      returnVal = 1;
+    } finally {
+      try {
+        if (ctxCreated) {
+          ctx.clear();
+        }
+        if (rj != null) {
+          if (returnVal != 0) {
+            rj.killJob();
+          }
+          HadoopJobExecHelper.runningJobKillURIs.remove(rj.getJobID());
+          jobID = rj.getID().toString();
+        }
+        ColumnTruncateMapper.jobClose(outputPath, success, job, console,
+          work.getDynPartCtx(), null);
+      } catch (Exception e) {
+      }
+    }
+
+    return (returnVal);
+  }
+
+  private void addInputPaths(JobConf job, ColumnTruncateWork work) {
+    FileInputFormat.addInputPath(job, new Path(work.getInputDir()));
+  }
+
+  @Override
+  public String getName() {
+    return "RCFile ColumnTruncate";
+  }
+
+  @Override
+  public StageType getType() {
+    return StageType.MAPRED;
+  }
+
+  @Override
+  public boolean checkFatalErrors(Counters ctrs, StringBuilder errMsg) {
+    return false;
+  }
+
+  @Override
+  public void logPlanProgress(SessionState ss) throws IOException {
+    // no op
+  }
+
+  @Override
+  public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
+    // no op
+  }
+
+  @Override
+  protected void localizeMRTmpFilesImpl(Context ctx) {
+    // no op
+  }
+
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java Thu Apr 25 08:03:48 2013
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io.rcfile.truncate;
+
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+import org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat;
+import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
+import org.apache.hadoop.hive.ql.plan.Explain;
+import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.mapred.Mapper;
+
+@Explain(displayName = "Column Truncate")
+public class ColumnTruncateWork extends MapredWork implements Serializable {
+
+  private static final long serialVersionUID = 1L;
+
+  private String inputDir;
+  private String outputDir;
+  private boolean hasDynamicPartitions;
+  private DynamicPartitionCtx dynPartCtx;
+  private boolean isListBucketingAlterTableConcatenate;
+  private ListBucketingCtx listBucketingCtx;
+  private List<Integer> droppedColumns;
+
+  public ColumnTruncateWork() {
+  }
+
+  public ColumnTruncateWork(List<Integer> droppedColumns, String inputDir, String outputDir) {
+    this(droppedColumns, inputDir, outputDir, false, null);
+  }
+
+  public ColumnTruncateWork(List<Integer> droppedColumns, String inputDir, String outputDir,
+      boolean hasDynamicPartitions, DynamicPartitionCtx dynPartCtx) {
+    super();
+    this.droppedColumns = droppedColumns;
+    this.inputDir = inputDir;
+    this.outputDir = outputDir;
+    this.hasDynamicPartitions = hasDynamicPartitions;
+    this.dynPartCtx = dynPartCtx;
+    PartitionDesc partDesc = new PartitionDesc();
+    partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class);
+    if(this.getPathToPartitionInfo() == null) {
+      this.setPathToPartitionInfo(new LinkedHashMap<String, PartitionDesc>());
+    }
+    if(this.getNumReduceTasks() == null) {
+      this.setNumReduceTasks(0);
+    }
+    this.getPathToPartitionInfo().put(inputDir, partDesc);
+  }
+
+  public String getInputDir() {
+    return inputDir;
+  }
+
+  public void setInputPaths(String inputDir) {
+    this.inputDir = inputDir;
+  }
+
+  public String getOutputDir() {
+    return outputDir;
+  }
+
+  public void setOutputDir(String outputDir) {
+    this.outputDir = outputDir;
+  }
+
+  public Class<? extends Mapper> getMapperClass() {
+    return ColumnTruncateMapper.class;
+  }
+
+  @Override
+  public Long getMinSplitSize() {
+    return null;
+  }
+
+  @Override
+  public String getInputformat() {
+    return BucketizedHiveInputFormat.class.getName();
+  }
+
+  @Override
+  public boolean isGatheringStats() {
+    return false;
+  }
+
+  public boolean hasDynamicPartitions() {
+    return this.hasDynamicPartitions;
+  }
+
+  public void setHasDynamicPartitions(boolean hasDynamicPartitions) {
+    this.hasDynamicPartitions = hasDynamicPartitions;
+  }
+
+  public DynamicPartitionCtx getDynPartCtx() {
+    return dynPartCtx;
+  }
+
+  public void setDynPartCtx(DynamicPartitionCtx dynPartCtx) {
+    this.dynPartCtx = dynPartCtx;
+  }
+
+  /**
+   * @return the listBucketingCtx
+   */
+  public ListBucketingCtx getListBucketingCtx() {
+    return listBucketingCtx;
+  }
+
+  /**
+   * @param listBucketingCtx the listBucketingCtx to set
+   */
+  public void setListBucketingCtx(ListBucketingCtx listBucketingCtx) {
+    this.listBucketingCtx = listBucketingCtx;
+  }
+
+  /**
+   * @return the isListBucketingAlterTableConcatenate
+   */
+  public boolean isListBucketingAlterTableConcatenate() {
+    return isListBucketingAlterTableConcatenate;
+  }
+
+  public List<Integer> getDroppedColumns() {
+    return droppedColumns;
+  }
+
+  public void setDroppedColumns(List<Integer> droppedColumns) {
+    this.droppedColumns = droppedColumns;
+  }
+
+}

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Thu Apr 25 08:03:48 2013
@@ -781,7 +781,148 @@ public class DDLSemanticAnalyzer extends
     }
 
     TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName, partSpec);
-    rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), truncateTblDesc), conf));
+
+    DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), truncateTblDesc);
+    Task<? extends Serializable> truncateTask = TaskFactory.get(ddlWork, conf);
+
+    // Is this a truncate column command
+    List<String> columnNames = null;
+    if (ast.getChildCount() == 2) {
+      try {
+        columnNames = getColumnNames((ASTNode)ast.getChild(1));
+
+        // Throw an error if the table is indexed
+        List<Index> indexes = db.getIndexes(table.getDbName(), tableName, (short)1);
+        if (indexes != null && indexes.size() > 0) {
+          throw new SemanticException(ErrorMsg.TRUNCATE_COLUMN_INDEXED_TABLE.getMsg());
+        }
+
+        List<String> bucketCols = null;
+        Class<? extends InputFormat> inputFormatClass = null;
+        boolean isArchived = false;
+        Path newTblPartLoc = null;
+        Path oldTblPartLoc = null;
+        List<FieldSchema> cols = null;
+        ListBucketingCtx lbCtx = null;
+        boolean isListBucketed = false;
+        List<String> listBucketColNames = null;
+
+        if (table.isPartitioned()) {
+          Partition part = db.getPartition(table, partSpec, false);
+
+          Path tabPath = table.getPath();
+          Path partPath = part.getPartitionPath();
+
+          // if the table is in a different dfs than the partition,
+          // replace the partition's dfs with the table's dfs.
+          newTblPartLoc = new Path(tabPath.toUri().getScheme(), tabPath.toUri()
+              .getAuthority(), partPath.toUri().getPath());
+
+          oldTblPartLoc = partPath;
+
+          cols = part.getCols();
+          bucketCols = part.getBucketCols();
+          inputFormatClass = part.getInputFormatClass();
+          isArchived = ArchiveUtils.isArchived(part);
+          lbCtx = constructListBucketingCtx(part.getSkewedColNames(), part.getSkewedColValues(),
+              part.getSkewedColValueLocationMaps(), part.isStoredAsSubDirectories(), conf);
+          isListBucketed = part.isStoredAsSubDirectories();
+          listBucketColNames = part.getSkewedColNames();
+        } else {
+          // input and output are the same
+          oldTblPartLoc = table.getPath();
+          newTblPartLoc = table.getPath();
+          cols  = table.getCols();
+          bucketCols = table.getBucketCols();
+          inputFormatClass = table.getInputFormatClass();
+          lbCtx = constructListBucketingCtx(table.getSkewedColNames(), table.getSkewedColValues(),
+              table.getSkewedColValueLocationMaps(), table.isStoredAsSubDirectories(), conf);
+          isListBucketed = table.isStoredAsSubDirectories();
+          listBucketColNames = table.getSkewedColNames();
+        }
+
+        // throw a HiveException for non-rcfile.
+        if (!inputFormatClass.equals(RCFileInputFormat.class)) {
+          throw new SemanticException(ErrorMsg.TRUNCATE_COLUMN_NOT_RC.getMsg());
+        }
+
+        // throw a HiveException if the table/partition is archived
+        if (isArchived) {
+          throw new SemanticException(ErrorMsg.TRUNCATE_COLUMN_ARCHIVED.getMsg());
+        }
+
+        Set<Integer> columnIndexes = new HashSet<Integer>();
+        for (String columnName : columnNames) {
+          boolean found = false;
+          for (int columnIndex = 0; columnIndex < cols.size(); columnIndex++) {
+            if (columnName.equalsIgnoreCase(cols.get(columnIndex).getName())) {
+              columnIndexes.add(columnIndex);
+              found = true;
+              break;
+            }
+          }
+          // Throw an exception if the user is trying to truncate a column which doesn't exist
+          if (!found) {
+            throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(columnName));
+          }
+          // Throw an exception if the table/partition is bucketed on one of the columns
+          for (String bucketCol : bucketCols) {
+            if (bucketCol.equalsIgnoreCase(columnName)) {
+              throw new SemanticException(ErrorMsg.TRUNCATE_BUCKETED_COLUMN.getMsg(columnName));
+            }
+          }
+          if (isListBucketed) {
+            for (String listBucketCol : listBucketColNames) {
+              if (listBucketCol.equalsIgnoreCase(columnName)) {
+                throw new SemanticException(
+                    ErrorMsg.TRUNCATE_LIST_BUCKETED_COLUMN.getMsg(columnName));
+              }
+            }
+          }
+        }
+
+        truncateTblDesc.setColumnIndexes(new ArrayList<Integer>(columnIndexes));
+
+        truncateTblDesc.setInputDir(oldTblPartLoc.toString());
+        addInputsOutputsAlterTable(tableName, partSpec);
+
+        truncateTblDesc.setLbCtx(lbCtx);
+
+        addInputsOutputsAlterTable(tableName, partSpec);
+        ddlWork.setNeedLock(true);
+        TableDesc tblDesc = Utilities.getTableDesc(table);
+        // Write the output to temporary directory and move it to the final location at the end
+        // so the operation is atomic.
+        String queryTmpdir = ctx.getExternalTmpFileURI(newTblPartLoc.toUri());
+        truncateTblDesc.setOutputDir(queryTmpdir);
+        LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, queryTmpdir, tblDesc,
+            partSpec == null ? new HashMap<String, String>() : partSpec);
+        ltd.setLbCtx(lbCtx);
+        Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
+            conf);
+        truncateTask.addDependentTask(moveTsk);
+
+        // Recalculate the HDFS stats if auto gather stats is set
+        if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
+          StatsWork statDesc;
+          if (oldTblPartLoc.equals(newTblPartLoc)) {
+            // If we're merging to the same location, we can avoid some metastore calls
+            tableSpec tablepart = new tableSpec(this.db, conf, root);
+            statDesc = new StatsWork(tablepart);
+          } else {
+            statDesc = new StatsWork(ltd);
+          }
+          statDesc.setNoStatsAggregator(true);
+          statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
+          Task<? extends Serializable> statTask = TaskFactory.get(statDesc, conf);
+          moveTsk.addDependentTask(statTask);
+        }
+      } catch (HiveException e) {
+        throw new SemanticException(e);
+      }
+    }
+
+    rootTasks.add(truncateTask);
   }
 
   private boolean isFullSpec(Table table, Map<String, String> partSpec) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Thu Apr 25 08:03:48 2013
@@ -756,7 +756,7 @@ createTableStatement
 truncateTableStatement
 @init { msgs.push("truncate table statement"); }
 @after { msgs.pop(); }
-    : KW_TRUNCATE KW_TABLE tablePartitionPrefix -> ^(TOK_TRUNCATETABLE tablePartitionPrefix);
+    : KW_TRUNCATE KW_TABLE tablePartitionPrefix (KW_COLUMNS LPAREN columnNameList RPAREN)? -> ^(TOK_TRUNCATETABLE tablePartitionPrefix columnNameList?);
 
 createIndexStatement
 @init { msgs.push("create index statement");}

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java?rev=1475662&r1=1475661&r2=1475662&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java Thu Apr 25 08:03:48 2013
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.plan;
 
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -30,6 +31,10 @@ public class TruncateTableDesc extends D
 
   private String tableName;
   private Map<String, String> partSpec;
+  private List<Integer> columnIndexes;
+  private String inputDir;
+  private String outputDir;
+  private ListBucketingCtx lbCtx;
 
   public TruncateTableDesc() {
   }
@@ -56,4 +61,37 @@ public class TruncateTableDesc extends D
   public void setPartSpec(Map<String, String> partSpec) {
     this.partSpec = partSpec;
   }
+
+  @Explain(displayName = "Column Indexes")
+  public List<Integer> getColumnIndexes() {
+    return columnIndexes;
+  }
+
+  public void setColumnIndexes(List<Integer> columnIndexes) {
+    this.columnIndexes = columnIndexes;
+  }
+
+  public String getInputDir() {
+    return inputDir;
+  }
+
+  public void setInputDir(String inputDir) {
+    this.inputDir = inputDir;
+  }
+
+  public String getOutputDir() {
+    return outputDir;
+  }
+
+  public void setOutputDir(String outputDir) {
+    this.outputDir = outputDir;
+  }
+
+  public ListBucketingCtx getLbCtx() {
+    return lbCtx;
+  }
+
+  public void setLbCtx(ListBucketingCtx lbCtx) {
+    this.lbCtx = lbCtx;
+  }
 }

Added: hive/trunk/ql/src/test/queries/clientnegative/truncate_bucketed_column.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/truncate_bucketed_column.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/truncate_bucketed_column.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/truncate_bucketed_column.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,7 @@
+-- Tests truncating a bucketed column
+
+CREATE TABLE test_tab (key STRING, value STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS RCFILE;
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src;
+
+TRUNCATE TABLE test_tab COLUMNS (key);

Added: hive/trunk/ql/src/test/queries/clientnegative/truncate_column_indexed_table.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/truncate_column_indexed_table.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/truncate_column_indexed_table.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/truncate_column_indexed_table.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,9 @@
+-- Tests truncating a column from an indexed table
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE;
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src;
+
+CREATE INDEX test_tab_index ON TABLE test_tab (key) as 'COMPACT' WITH DEFERRED REBUILD;
+
+TRUNCATE TABLE test_tab COLUMNS (value);

Added: hive/trunk/ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/truncate_column_list_bucketing.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,14 @@
+set hive.mapred.supports.subdirectories=true;
+set mapred.input.dir.recursive=true;
+
+-- Tests truncating a column on which a table is list bucketed
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE;
+
+ALTER TABLE test_tab
+SKEWED BY (key) ON ("484")
+STORED AS DIRECTORIES;
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src;
+
+TRUNCATE TABLE test_tab COLUMNS (key);

Added: hive/trunk/ql/src/test/queries/clientnegative/truncate_column_seqfile.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/truncate_column_seqfile.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/truncate_column_seqfile.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/truncate_column_seqfile.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,7 @@
+-- Tests truncating a column from a table stored as a sequence file
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS SEQUENCEFILE;
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src;
+
+TRUNCATE TABLE test_tab COLUMNS (key);

Added: hive/trunk/ql/src/test/queries/clientnegative/truncate_nonexistant_column.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/truncate_nonexistant_column.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/truncate_nonexistant_column.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/truncate_nonexistant_column.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,7 @@
+-- Tests attempting to truncate a column in a table that doesn't exist
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE;
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src;
+
+TRUNCATE TABLE test_tab COLUMNS (doesnt_exist);

Added: hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,7 @@
+-- Tests truncating a partition column
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE;
+
+INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src;
+
+TRUNCATE TABLE test_tab COLUMNS (part);

Added: hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column2.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column2.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/truncate_partition_column2.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,7 @@
+-- Tests truncating a partition column
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE;
+
+INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src;
+
+TRUNCATE TABLE test_tab PARTITION (part = '1') COLUMNS (part);

Added: hive/trunk/ql/src/test/queries/clientpositive/truncate_column.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/truncate_column.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/truncate_column.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/truncate_column.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,79 @@
+-- Tests truncating column(s) from a table, also tests that stats are updated
+
+CREATE TABLE test_tab (key STRING, value STRING) 
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' STORED AS RCFILE;
+
+set hive.stats.autogather=true;
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src LIMIT 10;
+
+DESC FORMATTED test_tab;
+
+SELECT * FROM test_tab;
+
+-- Truncate 1 column
+TRUNCATE TABLE test_tab COLUMNS (key);
+
+DESC FORMATTED test_tab;
+
+-- First column should be null
+SELECT * FROM test_tab;
+
+-- Truncate multiple columns
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src LIMIT 10;
+
+TRUNCATE TABLE test_tab COLUMNS (key, value);
+
+DESC FORMATTED test_tab;
+
+-- Both columns should be null
+SELECT * FROM test_tab;
+
+-- Truncate columns again
+TRUNCATE TABLE test_tab COLUMNS (key, value);
+
+DESC FORMATTED test_tab;
+
+-- Both columns should be null
+SELECT * FROM test_tab;
+
+-- Test truncating with a binary serde
+ALTER TABLE test_tab SET SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe';
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src LIMIT 10;
+
+DESC FORMATTED test_tab;
+
+SELECT * FROM test_tab;
+
+-- Truncate 1 column
+TRUNCATE TABLE test_tab COLUMNS (key);
+
+DESC FORMATTED test_tab;
+
+-- First column should be null
+SELECT * FROM test_tab;
+
+-- Truncate 2 columns
+TRUNCATE TABLE test_tab COLUMNS (key, value);
+
+DESC FORMATTED test_tab;
+
+-- Both columns should be null
+SELECT * FROM test_tab;
+
+-- Test truncating a partition
+CREATE TABLE test_tab_part (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE;
+
+INSERT OVERWRITE TABLE test_tab_part PARTITION (part = '1') SELECT * FROM src LIMIT 10;
+
+DESC FORMATTED test_tab_part PARTITION (part = '1');
+
+SELECT * FROM test_tab_part WHERE part = '1';
+
+TRUNCATE TABLE test_tab_part PARTITION (part = '1') COLUMNS (key);
+
+DESC FORMATTED test_tab_part PARTITION (part = '1');
+
+-- First column should be null
+SELECT * FROM test_tab_part WHERE part = '1';

Added: hive/trunk/ql/src/test/queries/clientpositive/truncate_column_buckets.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/truncate_column_buckets.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/truncate_column_buckets.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/truncate_column_buckets.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,23 @@
+-- Tests truncating columns from a bucketed table, table should remain bucketed
+
+CREATE TABLE test_tab (key STRING, value STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS RCFILE;
+
+set hive.enforce.bucketing=true;
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src;
+
+-- Check how many rows there are in each bucket, there should be two rows
+SELECT cnt FROM (
+SELECT INPUT__FILE__NAME file_name, count(*) cnt FROM 
+test_tab GROUP BY INPUT__FILE__NAME
+ORDER BY file_name DESC)a;
+
+-- Truncate a column on which the table is not bucketed
+TRUNCATE TABLE test_tab COLUMNS (value);
+
+-- Check how many rows there are in each bucket, this should produce the same rows as before
+-- because truncate should not break bucketing
+SELECT cnt FROM (
+SELECT INPUT__FILE__NAME file_name, count(*) cnt FROM 
+test_tab GROUP BY INPUT__FILE__NAME
+ORDER BY file_name DESC)a;

Added: hive/trunk/ql/src/test/queries/clientpositive/truncate_column_list_bucket.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/truncate_column_list_bucket.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/truncate_column_list_bucket.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/truncate_column_list_bucket.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,33 @@
+set hive.mapred.supports.subdirectories=true;
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+set hive.merge.mapfiles=false;	
+set hive.merge.mapredfiles=false; 
+set mapred.input.dir.recursive=true;
+
+-- Tests truncating a column from a list bucketing table
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE;
+
+ALTER TABLE test_tab
+SKEWED BY (key) ON ("484")
+STORED AS DIRECTORIES;
+
+INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src;
+
+set hive.optimize.listbucketing=true;
+SELECT * FROM test_tab WHERE part = '1' AND key = '0';
+
+TRUNCATE TABLE test_tab PARTITION (part ='1') COLUMNS (value);
+
+-- In the following select statements the list bucketing optimization should still be used
+-- In both cases value should be null
+
+EXPLAIN EXTENDED SELECT * FROM test_tab WHERE part = '1' AND key = '484';
+
+SELECT * FROM test_tab WHERE part = '1' AND key = '484';
+
+EXPLAIN EXTENDED SELECT * FROM test_tab WHERE part = '1' AND key = '0';
+
+SELECT * FROM test_tab WHERE part = '1' AND key = '0';

Added: hive/trunk/ql/src/test/queries/clientpositive/truncate_column_merge.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/truncate_column_merge.q?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/truncate_column_merge.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/truncate_column_merge.q Thu Apr 25 08:03:48 2013
@@ -0,0 +1,20 @@
+-- Tests truncating a column from a table with multiple files, then merging those files
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE;
+
+INSERT OVERWRITE TABLE test_tab SELECT * FROM src LIMIT 5;
+
+INSERT INTO TABLE test_tab SELECT * FROM src LIMIT 5;
+
+-- The value should be 2 indicating the table has 2 files
+SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM test_tab;
+
+TRUNCATE TABLE test_tab COLUMNS (key);
+
+ALTER TABLE test_tab CONCATENATE;
+
+-- The first column (key) should be null for all 10 rows
+SELECT * FROM test_tab;
+
+-- The value should be 1 indicating the table has 1 file
+SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM test_tab;

Added: hive/trunk/ql/src/test/results/clientnegative/truncate_bucketed_column.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_bucketed_column.q.out?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_bucketed_column.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_bucketed_column.q.out Thu Apr 25 08:03:48 2013
@@ -0,0 +1,20 @@
+PREHOOK: query: -- Tests truncating a bucketed column
+
+CREATE TABLE test_tab (key STRING, value STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests truncating a bucketed column
+
+CREATE TABLE test_tab (key STRING, value STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_tab
+POSTHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test_tab
+POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: A column on which a partition/table is bucketed cannot be truncated. key

Added: hive/trunk/ql/src/test/results/clientnegative/truncate_column_archived.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_column_archived.q.out?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_column_archived.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_column_archived.q.out Thu Apr 25 08:03:48 2013
@@ -0,0 +1,20 @@
+PREHOOK: query: -- Tests truncating a column from an archived partition
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests truncating a column from an archived partition
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_tab@part=1
+POSTHOOK: query: INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test_tab@part=1
+POSTHOOK: Lineage: test_tab PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException [Error 10107]: Archiving methods are currently disabled. Please see the Hive wiki for more information about enabling archiving

Added: hive/trunk/ql/src/test/results/clientnegative/truncate_column_indexed_table.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_column_indexed_table.q.out?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_column_indexed_table.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_column_indexed_table.q.out Thu Apr 25 08:03:48 2013
@@ -0,0 +1,27 @@
+PREHOOK: query: -- Tests truncating a column from an indexed table
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests truncating a column from an indexed table
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_tab
+POSTHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test_tab
+POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: CREATE INDEX test_tab_index ON TABLE test_tab (key) as 'COMPACT' WITH DEFERRED REBUILD
+PREHOOK: type: CREATEINDEX
+POSTHOOK: query: CREATE INDEX test_tab_index ON TABLE test_tab (key) as 'COMPACT' WITH DEFERRED REBUILD
+POSTHOOK: type: CREATEINDEX
+POSTHOOK: Output: default@default__test_tab_test_tab_index__
+POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Can not truncate columns from table with indexes

Added: hive/trunk/ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_column_list_bucketing.q.out Thu Apr 25 08:03:48 2013
@@ -0,0 +1,32 @@
+PREHOOK: query: -- Tests truncating a column on which a table is list bucketed
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests truncating a column on which a table is list bucketed
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: ALTER TABLE test_tab
+SKEWED BY (key) ON ("484")
+STORED AS DIRECTORIES
+PREHOOK: type: ALTERTABLE_SKEWED
+PREHOOK: Input: default@test_tab
+PREHOOK: Output: default@test_tab
+POSTHOOK: query: ALTER TABLE test_tab
+SKEWED BY (key) ON ("484")
+STORED AS DIRECTORIES
+POSTHOOK: type: ALTERTABLE_SKEWED
+POSTHOOK: Input: default@test_tab
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_tab
+POSTHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test_tab
+POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: A column on which a partition/table is list bucketed cannot be truncated. key

Added: hive/trunk/ql/src/test/results/clientnegative/truncate_column_seqfile.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_column_seqfile.q.out?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_column_seqfile.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_column_seqfile.q.out Thu Apr 25 08:03:48 2013
@@ -0,0 +1,20 @@
+PREHOOK: query: -- Tests truncating a column from a table stored as a sequence file
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS SEQUENCEFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests truncating a column from a table stored as a sequence file
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS SEQUENCEFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_tab
+POSTHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test_tab
+POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Only RCFileFormat supports column truncation.

Added: hive/trunk/ql/src/test/results/clientnegative/truncate_nonexistant_column.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_nonexistant_column.q.out?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_nonexistant_column.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_nonexistant_column.q.out Thu Apr 25 08:03:48 2013
@@ -0,0 +1,20 @@
+PREHOOK: query: -- Tests attempting to truncate a column in a table that doesn't exist
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests attempting to truncate a column in a table that doesn't exist
+
+CREATE TABLE test_tab (key STRING, value STRING) STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_tab
+POSTHOOK: query: INSERT OVERWRITE TABLE test_tab SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test_tab
+POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Invalid column reference doesnt_exist

Added: hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column.q.out Thu Apr 25 08:03:48 2013
@@ -0,0 +1,20 @@
+PREHOOK: query: -- Tests truncating a partition column
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests truncating a partition column
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_tab@part=1
+POSTHOOK: query: INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test_tab@part=1
+POSTHOOK: Lineage: test_tab PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException org.apache.hadoop.hive.ql.metadata.HiveException: table is partitioned but partition spec is not specified or does not fully match table partitioning: null

Added: hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column2.q.out?rev=1475662&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/truncate_partition_column2.q.out Thu Apr 25 08:03:48 2013
@@ -0,0 +1,20 @@
+PREHOOK: query: -- Tests truncating a partition column
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests truncating a partition column
+
+CREATE TABLE test_tab (key STRING, value STRING) PARTITIONED BY (part STRING) STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_tab
+PREHOOK: query: INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test_tab@part=1
+POSTHOOK: query: INSERT OVERWRITE TABLE test_tab PARTITION (part = '1') SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test_tab@part=1
+POSTHOOK: Lineage: test_tab PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test_tab PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Invalid column reference part



Mime
View raw message