hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject [1/7] hive git commit: HIVE-16171 : Support replication of truncate table (Sankar Hariappan, reviewed by Sushanth Sowmyan)
Date Fri, 28 Apr 2017 04:26:14 GMT
Repository: hive
Updated Branches:
  refs/heads/master fefeb2ac1 -> bbf5eccee


http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java
index 8edb50b..e5b8495 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterPartitionEvent.java
@@ -27,13 +27,15 @@ public class AlterPartitionEvent extends ListenerEvent {
   private final Partition oldPart;
   private final Partition newPart;
   private final Table table;
+  private final boolean isTruncateOp;
 
-  public AlterPartitionEvent(Partition oldPart, Partition newPart, Table table,
-      boolean status, HMSHandler handler) {
+  public AlterPartitionEvent(Partition oldPart, Partition newPart, Table table, boolean isTruncateOp,
+                             boolean status, HMSHandler handler) {
     super(status, handler);
     this.oldPart = oldPart;
     this.newPart = newPart;
     this.table = table;
+    this.isTruncateOp = isTruncateOp;
   }
 
   /**
@@ -58,4 +60,12 @@ public class AlterPartitionEvent extends ListenerEvent {
   public Table getTable() {
     return table;
   }
+
+  /**
+   * Get the truncate table flag
+   * @return
+   */
+  public boolean getIsTruncateOp() {
+    return isTruncateOp;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java
index 4d6dce2..22ea513 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AlterTableEvent.java
@@ -26,10 +26,13 @@ public class AlterTableEvent extends ListenerEvent {
 
   private final Table newTable;
   private final Table oldTable;
-  public AlterTableEvent (Table oldTable, Table newTable, boolean status, HMSHandler handler)
{
+  private final boolean isTruncateOp;
+
+  public AlterTableEvent (Table oldTable, Table newTable, boolean isTruncateOp, boolean status,
HMSHandler handler) {
     super (status, handler);
     this.oldTable = oldTable;
     this.newTable = newTable;
+    this.isTruncateOp = isTruncateOp;
   }
 
   /**
@@ -45,4 +48,11 @@ public class AlterTableEvent extends ListenerEvent {
   public Table getNewTable() {
     return newTable;
   }
+
+  /**
+   * @return the flag for truncate
+   */
+  public boolean getIsTruncateOp() {
+    return isTruncateOp;
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java
index ed6080b..e9ed7e5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterPartitionMessage.java
@@ -31,6 +31,8 @@ public abstract class AlterPartitionMessage extends EventMessage {
 
   public abstract String getTable();
 
+  public abstract boolean getIsTruncateOp();
+
   public abstract Map<String,String> getKeyValues();
 
   public abstract Table getTableObj() throws Exception;

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java
index 5487123..39a87bc 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/AlterTableMessage.java
@@ -28,6 +28,8 @@ public abstract class AlterTableMessage extends EventMessage {
 
   public abstract String getTable();
 
+  public abstract boolean getIsTruncateOp();
+
   public abstract Table getTableObjBefore() throws Exception;
 
   public abstract Table getTableObjAfter() throws Exception;

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
index 1ed7cc5..1bd52a8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/MessageFactory.java
@@ -149,9 +149,10 @@ public abstract class MessageFactory {
    * and some are not yet supported.
    * @param before The table before the alter
    * @param after The table after the alter
+   * @param isTruncateOp Flag to denote truncate table
    * @return
    */
-  public abstract AlterTableMessage buildAlterTableMessage(Table before, Table after);
+  public abstract AlterTableMessage buildAlterTableMessage(Table before, Table after, boolean
isTruncateOp);
 
   /**
    * Factory method for DropTableMessage.
@@ -175,10 +176,11 @@ public abstract class MessageFactory {
    * @param table The table in which the partition is being altered
    * @param before The partition before it was altered
    * @param after The partition after it was altered
+   * @param isTruncateOp Flag to denote truncate partition
    * @return a new AlterPartitionMessage
    */
   public abstract AlterPartitionMessage buildAlterPartitionMessage(Table table, Partition
before,
-                                                                   Partition after);
+                                                                   Partition after, boolean
isTruncateOp);
 
   /**
    * Factory method for DropPartitionMessage.

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
index dd1bf3c..bd7776c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterPartitionMessage.java
@@ -37,6 +37,9 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage {
   String server, servicePrincipal, db, table, tableObjJson;
 
   @JsonProperty
+  String isTruncateOp;
+
+  @JsonProperty
   Long timestamp;
 
   @JsonProperty
@@ -52,11 +55,12 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage {
   }
 
   public JSONAlterPartitionMessage(String server, String servicePrincipal, Table tableObj,
-      Partition partitionObjBefore, Partition partitionObjAfter, Long timestamp) {
+      Partition partitionObjBefore, Partition partitionObjAfter, boolean isTruncateOp, Long
timestamp) {
     this.server = server;
     this.servicePrincipal = servicePrincipal;
     this.db = tableObj.getDbName();
     this.table = tableObj.getTableName();
+    this.isTruncateOp = Boolean.toString(isTruncateOp);
     this.timestamp = timestamp;
     this.keyValues = JSONMessageFactory.getPartitionKeyValues(tableObj, partitionObjBefore);
     try {
@@ -95,6 +99,9 @@ public class JSONAlterPartitionMessage extends AlterPartitionMessage {
   }
 
   @Override
+  public boolean getIsTruncateOp() { return Boolean.parseBoolean(isTruncateOp); }
+
+  @Override
   public Map<String, String> getKeyValues() {
     return keyValues;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
index 792015e..58eb1a7 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONAlterTableMessage.java
@@ -32,6 +32,9 @@ public class JSONAlterTableMessage extends AlterTableMessage {
   String server, servicePrincipal, db, table, tableObjBeforeJson, tableObjAfterJson;
 
   @JsonProperty
+  String isTruncateOp;
+
+  @JsonProperty
   Long timestamp;
 
   /**
@@ -41,11 +44,12 @@ public class JSONAlterTableMessage extends AlterTableMessage {
   }
 
   public JSONAlterTableMessage(String server, String servicePrincipal, Table tableObjBefore,
Table tableObjAfter,
-      Long timestamp) {
+      boolean isTruncateOp, Long timestamp) {
     this.server = server;
     this.servicePrincipal = servicePrincipal;
     this.db = tableObjBefore.getDbName();
     this.table = tableObjBefore.getTableName();
+    this.isTruncateOp = Boolean.toString(isTruncateOp);
     this.timestamp = timestamp;
     try {
       this.tableObjBeforeJson = JSONMessageFactory.createTableObjJson(tableObjBefore);
@@ -82,6 +86,9 @@ public class JSONAlterTableMessage extends AlterTableMessage {
   }
 
   @Override
+  public boolean getIsTruncateOp() { return Boolean.parseBoolean(isTruncateOp); }
+
+  @Override
   public Table getTableObjBefore() throws Exception {
     return (Table) JSONMessageFactory.getTObj(tableObjBeforeJson,Table.class);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
index bb81949..04a4041 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java
@@ -28,6 +28,10 @@ import javax.annotation.Nullable;
 
 import com.google.common.collect.Iterables;
 
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.Function;
 import org.apache.hadoop.hive.metastore.api.Index;
@@ -104,8 +108,8 @@ public class JSONMessageFactory extends MessageFactory {
   }
 
   @Override
-  public AlterTableMessage buildAlterTableMessage(Table before, Table after) {
-    return new JSONAlterTableMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, before, after,
now());
+  public AlterTableMessage buildAlterTableMessage(Table before, Table after, boolean isTruncateOp)
{
+    return new JSONAlterTableMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, before, after,
isTruncateOp, now());
   }
 
   @Override
@@ -123,8 +127,8 @@ public class JSONMessageFactory extends MessageFactory {
 
   @Override
   public AlterPartitionMessage buildAlterPartitionMessage(Table table, Partition before,
-      Partition after) {
-    return new JSONAlterPartitionMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, table, before,
after,
+      Partition after, boolean isTruncateOp) {
+    return new JSONAlterPartitionMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, table, before,
after, isTruncateOp,
         now());
   }
 
@@ -297,5 +301,4 @@ public class JSONMessageFactory extends MessageFactory {
     };
     return getTObjs(Iterables.transform(jsonArrayIterator, textExtractor), objClass);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 917e565..757b7fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4661,32 +4661,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
     String tableName = truncateTableDesc.getTableName();
     Map<String, String> partSpec = truncateTableDesc.getPartSpec();
 
-    Table table = db.getTable(tableName, true);
-    boolean isAutopurge = "true".equalsIgnoreCase(table.getProperty("auto.purge"));
     try {
-      // this is not transactional
-      for (Path location : getLocations(db, table, partSpec)) {
-        FileSystem fs = location.getFileSystem(conf);
-        HadoopShims.HdfsEncryptionShim shim
-          = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf);
-        if (!shim.isPathEncrypted(location)) {
-          HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location);
-          FileStatus targetStatus = fs.getFileStatus(location);
-          String targetGroup = targetStatus == null ? null : targetStatus.getGroup();
-          FileUtils.moveToTrash(fs, location, conf, isAutopurge);
-          fs.mkdirs(location);
-          HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false);
-        } else {
-          FileStatus[] statuses = fs.listStatus(location, FileUtils.HIDDEN_FILES_PATH_FILTER);
-          if (statuses == null || statuses.length == 0) {
-            continue;
-          }
-          boolean success = Hive.trashFiles(fs, statuses, conf, isAutopurge);
-          if (!success) {
-            throw new HiveException("Error in deleting the contents of " + location.toString());
-          }
-        }
-      }
+      db.truncateTable(tableName, partSpec);
     } catch (Exception e) {
       throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
     }
@@ -4717,58 +4693,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
     return 0;
   }
 
-  private List<Path> getLocations(Hive db, Table table, Map<String, String> partSpec)
-      throws HiveException, InvalidOperationException {
-    List<Path> locations = new ArrayList<Path>();
-    if (partSpec == null) {
-      if (table.isPartitioned()) {
-        for (Partition partition : db.getPartitions(table)) {
-          locations.add(partition.getDataLocation());
-          EnvironmentContext environmentContext = new EnvironmentContext();
-          if (needToUpdateStats(partition.getParameters(), environmentContext)) {
-            db.alterPartition(table.getDbName(), table.getTableName(), partition, environmentContext);
-          }
-        }
-      } else {
-        locations.add(table.getPath());
-        EnvironmentContext environmentContext = new EnvironmentContext();
-        if (needToUpdateStats(table.getParameters(), environmentContext)) {
-          db.alterTable(table.getDbName()+"."+table.getTableName(), table, environmentContext);
-        }
-      }
-    } else {
-      for (Partition partition : db.getPartitionsByNames(table, partSpec)) {
-        locations.add(partition.getDataLocation());
-        EnvironmentContext environmentContext = new EnvironmentContext();
-        if (needToUpdateStats(partition.getParameters(), environmentContext)) {
-          db.alterPartition(table.getDbName(), table.getTableName(), partition, environmentContext);
-        }
-      }
-    }
-    return locations;
-  }
-
-  private boolean needToUpdateStats(Map<String,String> props, EnvironmentContext environmentContext)
{
-    if (null == props) {
-      return false;
-    }
-    boolean statsPresent = false;
-    for (String stat : StatsSetupConst.supportedStats) {
-      String statVal = props.get(stat);
-      if (statVal != null && Long.parseLong(statVal) > 0) {
-        statsPresent = true;
-        //In the case of truncate table, we set the stats to be 0.
-        props.put(stat, "0");
-      }
-    }
-    //first set basic stats to true
-    StatsSetupConst.setBasicStatsState(props, StatsSetupConst.TRUE);
-    environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK);
-    //then invalidate column stats
-    StatsSetupConst.clearColumnStatsState(props);
-    return statsPresent;
-  }
-
   @Override
   public StageType getType() {
     return StageType.DDL;

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ec36487..45c77a2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1201,6 +1201,27 @@ public class Hive {
     }
   }
 
+
+
+  /**
+   * Truncates the table/partition as per specifications. Just trash the data files
+   *
+   * @param dbDotTableName
+   *          name of the table
+   * @throws HiveException
+   */
+  public void truncateTable(String dbDotTableName, Map<String, String> partSpec) throws
HiveException {
+    try {
+      Table table = getTable(dbDotTableName, true);
+
+      List<String> partNames = ((null == partSpec)
+                       ? null : getPartitionNames(table.getDbName(), table.getTableName(),
partSpec, (short) -1));
+      getMSC().truncateTable(table.getDbName(), table.getTableName(), partNames);
+    } catch (Exception e) {
+      throw new HiveException(e);
+    }
+  }
+
   public HiveConf getConf() {
     return (conf);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 8752e51..71d6074 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -505,7 +505,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       }
     }
 
-    // Next, we verify that the destination table is not offline, a view, or a non-native
table
+    // Next, we verify that the destination table is not offline, or a non-native table
     EximUtil.validateTable(table);
 
     // If the import statement specified that we're importing to an external

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index a85ba42..c4cc9bc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.ql.plan.PlanUtils;
 import org.apache.hadoop.hive.ql.plan.RenamePartitionDesc;
+import org.apache.hadoop.hive.ql.plan.TruncateTableDesc;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.io.IOUtils;
@@ -128,8 +129,10 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer
{
     EVENT_DROP_PARTITION("EVENT_DROP_PARTITION"),
     EVENT_ALTER_TABLE("EVENT_ALTER_TABLE"),
     EVENT_RENAME_TABLE("EVENT_RENAME_TABLE"),
+    EVENT_TRUNCATE_TABLE("EVENT_TRUNCATE_TABLE"),
     EVENT_ALTER_PARTITION("EVENT_ALTER_PARTITION"),
     EVENT_RENAME_PARTITION("EVENT_RENAME_PARTITION"),
+    EVENT_TRUNCATE_PARTITION("EVENT_TRUNCATE_PARTITION"),
     EVENT_INSERT("EVENT_INSERT"),
     EVENT_UNKNOWN("EVENT_UNKNOWN");
 
@@ -937,6 +940,24 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer
{
           }
         }
       }
+      case EVENT_TRUNCATE_TABLE: {
+        AlterTableMessage truncateTableMessage = md.getAlterTableMessage(dmd.getPayload());
+        String actualDbName = ((dbName == null) || dbName.isEmpty() ? truncateTableMessage.getDB()
: dbName);
+        String actualTblName = ((tblName == null) || tblName.isEmpty() ? truncateTableMessage.getTable()
: tblName);
+
+        TruncateTableDesc truncateTableDesc = new TruncateTableDesc(
+                actualDbName + "." + actualTblName, null);
+        Task<DDLWork> truncateTableTask = TaskFactory.get(new DDLWork(inputs, outputs,
truncateTableDesc), conf);
+        if (precursor != null) {
+          precursor.addDependentTask(truncateTableTask);
+        }
+
+        List<Task<? extends Serializable>> tasks = new ArrayList<Task<?
extends Serializable>>();
+        tasks.add(truncateTableTask);
+        LOG.debug("Added truncate tbl task : {}:{}", truncateTableTask.getId(), truncateTableDesc.getTableName());
+        dbsUpdated.put(actualDbName,dmd.getEventTo());
+        return tasks;
+      }
       case EVENT_ALTER_PARTITION: {
         return analyzeTableLoad(dbName, tblName, locn, precursor, dbsUpdated, tablesUpdated);
       }
@@ -978,6 +999,40 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer
{
         tablesUpdated.put(tableName, dmd.getEventTo());
         return tasks;
       }
+      case EVENT_TRUNCATE_PARTITION: {
+        AlterPartitionMessage truncatePtnMessage = md.getAlterPartitionMessage(dmd.getPayload());
+        String actualDbName = ((dbName == null) || dbName.isEmpty() ? truncatePtnMessage.getDB()
: dbName);
+        String actualTblName = ((tblName == null) || tblName.isEmpty() ? truncatePtnMessage.getTable()
: tblName);
+
+        Map<String, String> partSpec = new LinkedHashMap<String,String>();
+        try {
+          org.apache.hadoop.hive.metastore.api.Table tblObj = truncatePtnMessage.getTableObj();
+          org.apache.hadoop.hive.metastore.api.Partition pobjAfter = truncatePtnMessage.getPtnObjAfter();
+          Iterator<String> afterValIter = pobjAfter.getValuesIterator();
+          for (FieldSchema fs : tblObj.getPartitionKeys()){
+            partSpec.put(fs.getName(), afterValIter.next());
+          }
+        } catch (Exception e) {
+          if (!(e instanceof SemanticException)){
+            throw new SemanticException("Error reading message members", e);
+          } else {
+            throw (SemanticException)e;
+          }
+        }
+
+        TruncateTableDesc truncateTableDesc = new TruncateTableDesc(
+                actualDbName + "." + actualTblName, partSpec);
+        Task<DDLWork> truncatePtnTask = TaskFactory.get(new DDLWork(inputs, outputs,
truncateTableDesc), conf);
+        if (precursor != null) {
+          precursor.addDependentTask(truncatePtnTask);
+        }
+
+        List<Task<? extends Serializable>> tasks = new ArrayList<Task<?
extends Serializable>>();
+        tasks.add(truncatePtnTask);
+        LOG.debug("Added truncate ptn task : {}:{}", truncatePtnTask.getId(), truncateTableDesc.getTableName());
+        dbsUpdated.put(actualDbName,dmd.getEventTo());
+        return tasks;
+      }
       case EVENT_INSERT: {
         md = MessageFactory.getInstance().getDeserializer();
         InsertMessage insertMessage = md.getInsertMessage(dmd.getPayload());

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
index 1073cd0..20d04dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
 
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -34,6 +35,7 @@ import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMe
 public class AlterPartitionHandler extends AbstractHandler {
   private final org.apache.hadoop.hive.metastore.api.Partition after;
   private final org.apache.hadoop.hive.metastore.api.Table tableObject;
+  private final boolean isTruncateOp;
   private final Scenario scenario;
 
   AlterPartitionHandler(NotificationEvent event) throws Exception {
@@ -42,6 +44,7 @@ public class AlterPartitionHandler extends AbstractHandler {
     tableObject = apm.getTableObj();
     org.apache.hadoop.hive.metastore.api.Partition before = apm.getPtnObjBefore();
     after = apm.getPtnObjAfter();
+    isTruncateOp = apm.getIsTruncateOp();
     scenario = scenarioType(before, after);
   }
 
@@ -57,12 +60,18 @@ public class AlterPartitionHandler extends AbstractHandler {
       DUMPTYPE dumpType() {
         return DUMPTYPE.EVENT_RENAME_PARTITION;
       }
+    },
+    TRUNCATE {
+      @Override
+      DUMPTYPE dumpType() {
+        return DUMPTYPE.EVENT_TRUNCATE_PARTITION;
+      }
     };
 
     abstract DUMPTYPE dumpType();
   }
 
-  private static Scenario scenarioType(org.apache.hadoop.hive.metastore.api.Partition before,
+  private Scenario scenarioType(org.apache.hadoop.hive.metastore.api.Partition before,
       org.apache.hadoop.hive.metastore.api.Partition after) {
     Iterator<String> beforeValIter = before.getValuesIterator();
     Iterator<String> afterValIter = after.getValuesIterator();
@@ -71,7 +80,7 @@ public class AlterPartitionHandler extends AbstractHandler {
         return Scenario.RENAME;
       }
     }
-    return Scenario.ALTER;
+    return isTruncateOp ? Scenario.TRUNCATE : Scenario.ALTER;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
index 04d9d79..bfe0181 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hive.metastore.api.NotificationEvent;
 import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
 
 import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
 import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
@@ -29,6 +30,7 @@ import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMe
 public class AlterTableHandler extends AbstractHandler {
   private final org.apache.hadoop.hive.metastore.api.Table before;
   private final org.apache.hadoop.hive.metastore.api.Table after;
+  private final boolean isTruncateOp;
   private final Scenario scenario;
 
   private enum Scenario {
@@ -43,6 +45,12 @@ public class AlterTableHandler extends AbstractHandler {
       DUMPTYPE dumpType() {
         return DUMPTYPE.EVENT_RENAME_TABLE;
       }
+    },
+    TRUNCATE {
+      @Override
+      DUMPTYPE dumpType() {
+        return DUMPTYPE.EVENT_TRUNCATE_TABLE;
+      }
     };
 
     abstract DUMPTYPE dumpType();
@@ -53,15 +61,18 @@ public class AlterTableHandler extends AbstractHandler {
     AlterTableMessage atm = deserializer.getAlterTableMessage(event.getMessage());
     before = atm.getTableObjBefore();
     after = atm.getTableObjAfter();
+    isTruncateOp = atm.getIsTruncateOp();
     scenario = scenarioType(before, after);
   }
 
-  private static Scenario scenarioType(org.apache.hadoop.hive.metastore.api.Table before,
+  private Scenario scenarioType(org.apache.hadoop.hive.metastore.api.Table before,
       org.apache.hadoop.hive.metastore.api.Table after) {
-    return before.getDbName().equals(after.getDbName())
-        && before.getTableName().equals(after.getTableName())
-        ? Scenario.ALTER
-        : Scenario.RENAME;
+    if (before.getDbName().equals(after.getDbName())
+        && before.getTableName().equals(after.getTableName())) {
+      return isTruncateOp ? Scenario.TRUNCATE : Scenario.ALTER;
+    } else {
+      return Scenario.RENAME;
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/bbf5ecce/ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out
b/ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out
index af21343..a7c9b3f 100644
--- a/ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out
+++ b/ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out
@@ -292,10 +292,10 @@ Table Type:         	MANAGED_TABLE
 Table Parameters:	 	 
 	COLUMN_STATS_ACCURATE	{\"BASIC_STATS\":\"true\"}
 #### A masked pattern was here ####
-	numFiles            	2                   
+	numFiles            	0                   
 	numRows             	0                   
 	rawDataSize         	0                   
-	totalSize           	547                 
+	totalSize           	0                   
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -322,11 +322,11 @@ STAGE PLANS:
       Map Operator Tree:
           TableScan
             alias: calendar
-            Statistics: Num rows: 136 Data size: 547 Basic stats: COMPLETE Column stats:
NONE
+            Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
             Select Operator
               expressions: month (type: int)
               outputColumnNames: month
-              Statistics: Num rows: 136 Data size: 547 Basic stats: COMPLETE Column stats:
NONE
+              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
               Group By Operator
                 aggregations: max(month)
                 mode: hash


Mime
View raw message