hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kevinwilf...@apache.org
Subject svn commit: r1332873 [1/4] - in /hive/trunk/contrib/src/java/org/apache/hadoop/hive: metastore/ metastore/hooks/ ql/ ql/hooks/ ql/hooks/conf/ ql/stats/
Date Tue, 01 May 2012 22:52:40 GMT
Author: kevinwilfong
Date: Tue May  1 22:52:38 2012
New Revision: 1332873

URL: http://svn.apache.org/viewvc?rev=1332873&view=rev
Log:
HIVE-2986 Create the hooks (njain via kevinwilfong)

Added:
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ExternalInputsHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/FbUpdateInputAccessTimeHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/FifoPoolHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/HiveConfigLoggingHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/HookUtils.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/JobStatsHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/JobTrackerHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/LineageHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/Pair.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/PyRulesHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/QueryDroppedPartitionsHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/QueryPlanHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/RegressionTestHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ReplicationHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SMCStatsDBHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SampleConcurrencyHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SmcConfigDriverRunHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SmcConfigHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SplitSizeHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/StartFinishHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/SuggestionPrintingHook.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/Triple.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/conf/
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/conf/FBHiveConf.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/stats/
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/stats/HiveStatsMetricsPublisher.java

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/AuditMetaStoreEventListener.java Tue May  1 22:52:38 2012
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent;
+import org.apache.hadoop.hive.ql.hooks.BaseReplicationHook;
+import org.apache.hadoop.hive.ql.hooks.ConnectionUrlFactory;
+import org.apache.hadoop.hive.ql.hooks.HookUtils;
+import org.apache.hadoop.hive.ql.hooks.ReplicationHook;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TSimpleJSONProtocol;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/*
+ * MetaStoreEventListener that logs metastore operations to the audit log.
+ * The operations that this listener logs are only those from the thrift server,
+ * and not the CLI, because ReplicationHook currently logs queries from the CLI.
+ */
+public class AuditMetaStoreEventListener extends MetaStoreEventListener {
+  public static final Log LOG = LogFactory.getLog(AuditMetaStoreEventListener.class);
+
+  private static final String COMMAND_TYPE = "METASTORE_API";
+  private static final String COMMAND_NAME = "name";
+  private static final String ADD_PARTITION_COMMAND = "ADD_PARTITION";
+  private static final String ALTER_PARTITION_COMMAND = "ALTER_PARTITION";
+  private static final String ALTER_TABLE_COMMAND = "ALTER_TABLE";
+  private static final String CREATE_TABLE_COMMAND = "CREATE_TABLE";
+  private static final String DROP_PARTITION_COMMAND = "DROP_PARTITION";
+  private static final String DROP_DATABASE_COMMAND = "DROP_DATABASE";
+  private static final String DROP_TABLE_COMMAND = "DROP_TABLE";
+  private static final String NEW_TABLE = "new_table";
+  private static final String OLD_TABLE = "old_table";
+  private static final String NEW_PARTITION = "new_partition";
+  private static final String OLD_PARTITION = "old_partition";
+
+  private final TSerializer jsonSerializer;
+
+
+  protected ConnectionUrlFactory urlFactory = null;
+
+  public AuditMetaStoreEventListener(Configuration config) throws Exception{
+    super(config);
+    urlFactory = BaseReplicationHook.getReplicationMySqlUrl();
+    jsonSerializer = new TSerializer(new TSimpleJSONProtocol.Factory());
+  }
+
+  private void insertToDB(Set<ReadEntity> inputs,
+      Set<WriteEntity> outputs, ListenerEvent event, String command) throws MetaException {
+    HiveConf conf = event.getHandler().getHiveConf();
+    //if HIVEQUERYID is set, then this command came from a CLI
+    // (and will execute posthooks).  We don't want to log such a command
+    if (conf.getVar(HiveConf.ConfVars.HIVEQUERYID) == null ||
+        conf.getVar(HiveConf.ConfVars.HIVEQUERYID).isEmpty()) {
+      try {
+        ArrayList<Object> sqlParams = new ArrayList<Object>();
+        sqlParams.add(command);
+        sqlParams.add(StringEscapeUtils.escapeJava(ReplicationHook.entitiesToString(inputs)));
+        sqlParams.add(ReplicationHook.entitiesToString(outputs));
+        sqlParams.add(COMMAND_TYPE);
+
+        // Assertion at beginning of method guarantees this string will remain empty
+        String sql = "insert into snc1_command_log " +
+                     " set command = ?, inputs = ?, outputs = ?, command_type = ?";
+
+        String ipAddress = HMSHandler.getIpAddress();
+        if (ipAddress != null) {
+          if (ipAddress.startsWith("/")) {
+            ipAddress = ipAddress.replaceFirst("/", "");
+          }
+          sql += ", user_info = ?";
+          sqlParams.add(ipAddress);
+        }
+
+        HookUtils.runInsert(conf, urlFactory, sql, sqlParams, HookUtils
+            .getSqlNumRetry(conf));
+      } catch (Exception e) {
+        throw new MetaException(e.getMessage());
+      }
+    }
+  }
+
+  private Table getTableFromPart(Partition p, ListenerEvent event) throws MetaException {
+    try {
+      return event.getHandler().get_table(p.getDbName(), p.getTableName());
+    } catch (Exception e) {
+      throw new MetaException(e.getMessage());
+    }
+  }
+
+  private org.apache.hadoop.hive.ql.metadata.Table getQlTable(Table t) {
+    return new org.apache.hadoop.hive.ql.metadata.Table(t);
+  }
+
+  private org.apache.hadoop.hive.ql.metadata.Partition getQlPartition(Table t, Partition p)
+      throws MetaException{
+    try {
+      org.apache.hadoop.hive.ql.metadata.Table qlTable = getQlTable(t);
+      return new org.apache.hadoop.hive.ql.metadata.Partition(qlTable, p);
+    } catch (HiveException e) {
+      throw new MetaException(e.getMessage());
+    }
+  }
+
+  private ReadEntity getPartitionInput(Partition p, ListenerEvent event)  throws MetaException {
+    Table mTable = getTableFromPart(p, event);
+    ReadEntity input = new ReadEntity(getQlPartition(mTable, p));
+    return input;
+  }
+
+  private WriteEntity getPartitionOutput(Partition p, ListenerEvent event)  throws MetaException {
+    try {
+      Table mTable = event.getHandler().get_table(p.getDbName(), p.getTableName());
+      WriteEntity output = new WriteEntity(getQlPartition(mTable, p));
+      return output;
+    } catch (Exception e) {
+      throw new MetaException(e.getMessage());
+    }
+  }
+
+  private void logNoSuccess() {
+    LOG.info("ListenerEvent success is false");
+  }
+
+  private void addCommandNameToCommand(JSONObject command, String name) {
+    try {
+      command.put(COMMAND_NAME, name);
+    } catch (JSONException e) {
+      LOG.error("Could not add command name to JSON object", e);
+    }
+  }
+
+  private void addTBaseToCommand(JSONObject command, TBase object, String objectName) {
+    try {
+      command.put(objectName, new JSONObject(jsonSerializer.toString(object)));
+    } catch (JSONException e) {
+      LOG.error("Could not add " + objectName + " to JSON object", e);
+    } catch (TException e) {
+      LOG.error("Could not serialize " + objectName + " to JSON", e);
+    }
+  }
+
+  @Override
+  public void onAddPartition(AddPartitionEvent event) throws MetaException {
+    if (!event.getStatus()) {
+      logNoSuccess();
+      return;
+    }
+
+    Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+    Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+    JSONObject command = new JSONObject();
+    inputs.add(new ReadEntity(getQlTable(getTableFromPart(event.getPartition(), event))));
+    outputs.add(getPartitionOutput(event.getPartition(), event));
+    addCommandNameToCommand(command, ADD_PARTITION_COMMAND);
+
+    insertToDB(inputs, outputs, event, command.toString());
+  }
+
+  @Override
+  public void onAlterPartition(AlterPartitionEvent event) throws MetaException {
+    if (!event.getStatus()) {
+      logNoSuccess();
+      return;
+    }
+
+    Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+    Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+    JSONObject command = new JSONObject();
+    inputs.add(getPartitionInput(event.getOldPartition(), event));
+    outputs.add(getPartitionOutput(event.getNewPartition(), event));
+    addCommandNameToCommand(command, ALTER_PARTITION_COMMAND);
+    addTBaseToCommand(command, event.getOldPartition(), OLD_PARTITION);
+    addTBaseToCommand(command, event.getNewPartition(), NEW_PARTITION);
+
+    insertToDB(inputs, outputs, event, command.toString());
+  }
+
+  @Override
+  public void onDropPartition(DropPartitionEvent event) throws MetaException {
+    if (!event.getStatus()) {
+      logNoSuccess();
+      return;
+    }
+
+    Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+    Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+    JSONObject command = new JSONObject();
+    inputs.add(new ReadEntity(getQlTable(getTableFromPart(event.getPartition(), event))));
+    outputs.add(getPartitionOutput(event.getPartition(), event));
+    addCommandNameToCommand(command, DROP_PARTITION_COMMAND);
+
+    insertToDB(inputs, outputs, event, command.toString());
+  }
+
+  @Override
+  /*
+   * Currently, on the create database CLI command, nothing gets logged.
+   */
+  public void onCreateDatabase(CreateDatabaseEvent event) throws MetaException {
+  }
+
+  @Override
+  public void onDropDatabase(DropDatabaseEvent event) throws MetaException {
+    if (!event.getStatus()) {
+      logNoSuccess();
+      return;
+    }
+
+    Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+    Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+    JSONObject command = new JSONObject();
+    addCommandNameToCommand(command, DROP_DATABASE_COMMAND);
+
+    insertToDB(inputs, outputs, event, command.toString());
+  }
+
+  @Override
+  public void onCreateTable(CreateTableEvent event) throws MetaException {
+    if (!event.getStatus()) {
+      logNoSuccess();
+      return;
+    }
+
+    Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+    Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+    JSONObject command = new JSONObject();
+    outputs.add(new WriteEntity(getQlTable(event.getTable())));
+    addCommandNameToCommand(command, CREATE_TABLE_COMMAND);
+
+    insertToDB(inputs, outputs, event, command.toString());
+  }
+
+  @Override
+  public void onDropTable(DropTableEvent event) throws MetaException {
+    if (!event.getStatus()) {
+      logNoSuccess();
+      return;
+    }
+
+    Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+    Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+    JSONObject command = new JSONObject();
+    inputs.add(new ReadEntity(getQlTable(event.getTable())));
+    outputs.add(new WriteEntity(getQlTable(event.getTable())));
+    addCommandNameToCommand(command, DROP_TABLE_COMMAND);
+
+    insertToDB(inputs, outputs, event, command.toString());
+  }
+
+  @Override
+  public void onAlterTable(AlterTableEvent event) throws MetaException {
+    if (!event.getStatus()) {
+      logNoSuccess();
+      return;
+    }
+
+    Set<ReadEntity> inputs = new HashSet<ReadEntity>();
+    Set<WriteEntity> outputs = new HashSet<WriteEntity>();
+    JSONObject command = new JSONObject();
+    inputs.add(new ReadEntity(getQlTable(event.getOldTable())));
+    outputs.add(new WriteEntity(getQlTable(event.getOldTable())));
+    outputs.add(new WriteEntity(getQlTable(event.getNewTable())));
+    addCommandNameToCommand(command, ALTER_TABLE_COMMAND);
+    addTBaseToCommand(command, event.getOldTable(), OLD_TABLE);
+    addTBaseToCommand(command, event.getNewTable(), NEW_TABLE);
+
+    insertToDB(inputs, outputs, event, command.toString());
+  }
+
+  @Override
+  public void onLoadPartitionDone(LoadPartitionDoneEvent lpe) throws MetaException {
+  }
+
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/CounterMetaStoreEndFunctionListener.java Tue May  1 22:52:38 2012
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import java.util.AbstractMap;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.MetaStoreEndFunctionContext;
+import org.apache.hadoop.hive.metastore.MetaStoreEndFunctionListener;
+import org.apache.hadoop.hive.ql.hooks.HookUtils;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+
+/*
+ * MetaStoreEndFunctionListener that uses the StatsManager to collect fb303 counters for
+ * the number of successes and failures for each metastore thrift function, bucketed by time.
+ */
+public class CounterMetaStoreEndFunctionListener extends MetaStoreEndFunctionListener {
+
+  StatsManager stats = null;
+
+  public CounterMetaStoreEndFunctionListener(Configuration config) {
+    super(config);
+    String statsMgr = config.get(FBHiveConf.METASTORE_LISTENER_STATS_MANAGER);
+    if ((statsMgr == null) || (statsMgr.isEmpty())) {
+      return;
+    }
+
+    stats = HookUtils.getObject(config, statsMgr);
+  }
+
+  @Override
+  public void onEndFunction(String functionName, MetaStoreEndFunctionContext context) {
+    if (stats == null) {
+      return;
+    }
+
+    // Construct the counter name, as <functionName> for success
+    // and <functionName.failure> for failure
+    String statName = functionName + (context.isSuccess() ? "" : ".failure");
+
+    // If this is the first time this counter name has been seen, initialize it
+    if (!stats.containsKey(statName)) {
+      stats.addCountStatType(statName);
+    }
+
+    stats.addStatValue(statName, 1);
+  }
+
+  @Override
+  public void exportCounters(AbstractMap<String, Long> counters) {
+    if (stats == null) {
+      return;
+    }
+
+    // For each counter the StatsManager has collected, add it to the map of fb303 counters
+    for (Entry<String, Long> entry : stats.getCounters().entrySet()) {
+      counters.put(entry.getKey(), entry.getValue());
+    }
+  }
+
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/FbhiveAlterHandler.java Tue May  1 22:52:38 2012
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.HiveAlterHandler;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.hooks.HookUtils;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+
+/*
+ * Subclass of HiveAlterHandler.  Checks that if the table, or partition's table has a
+ * creation_cluster set, that cluster matches the current cluster where the metastore is running.
+ * If so, or if the table or partition's table does not have creation_cluster set, it calls its
+ * super classes implementation of the alter method, i.e. it behaves normally.  If not, it throws
+ * a MetaException.
+ */
+public class FbhiveAlterHandler extends HiveAlterHandler {
+  public static final Log LOG = LogFactory.getLog(FbhiveAlterHandler.class);
+
+  @Override
+  public Partition alterPartition(RawStore ms, Warehouse wh, String dbname,
+      String name, List<String> part_vals, Partition new_part)
+      throws InvalidOperationException, InvalidObjectException,
+      AlreadyExistsException, MetaException {
+
+    String exception = "Partition in table " + name + " cannot be altered.";
+    checkTableCluster(ms, dbname, name, exception);
+
+    return super.alterPartition(ms, wh, dbname, name, part_vals, new_part);
+  }
+
+  @Override
+  public void alterTable(RawStore msdb, Warehouse wh, String dbname,
+      String name, Table newt) throws InvalidOperationException, MetaException {
+
+    String exception = "Table " + name + " cannot be altered.";
+    checkTableCluster(msdb, dbname, name, exception);
+
+    super.alterTable(msdb, wh, dbname, name, newt);
+  }
+
+  private void checkTableCluster(RawStore msdb, String dbName, String tableName,
+      String exception) throws MetaException{
+
+    Table oldt = msdb.getTable(dbName.toLowerCase(), tableName.toLowerCase());
+    if (oldt != null) {
+      String creationCluster = oldt.getParameters().get(HookUtils.TABLE_CREATION_CLUSTER);
+      String currentCluster = hiveConf.get(FBHiveConf.FB_CURRENT_CLUSTER);
+      if (creationCluster != null &&
+          currentCluster != null &&
+          !creationCluster.equals(currentCluster)) {
+        throw new MetaException(exception +
+            " Table's cluster is " + creationCluster + "," +
+            " whereas current package is " + currentCluster);
+      }
+    }
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/MysqlSmcHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.hooks.ConnectionUrlFactory;
+import org.apache.hadoop.hive.ql.hooks.HookUtils;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+
+public class MysqlSmcHook implements JDOConnectionURLHook {
+
+  static final private Log LOG =
+    LogFactory.getLog("hive.metastore.hooks.MysqlSmcHook");
+
+  ConnectionUrlFactory urlFactory = null;
+
+  @Override
+  public String getJdoConnectionUrl(Configuration conf)
+  throws Exception {
+
+    String smcUrl = conf.get(FBHiveConf.METASTORE_SMC_URL);
+    if (smcUrl == null) {
+      throw new Exception(FBHiveConf.METASTORE_SMC_URL + " is not defined");
+    }
+    String mysqlTier = conf.get(FBHiveConf.METASTORE_MYSQL_TIER_VAR_NAME);
+    if (mysqlTier == null) {
+      throw new Exception(FBHiveConf.METASTORE_MYSQL_TIER_VAR_NAME + " is not defined");
+    }
+    String mysqlProps = conf.get(FBHiveConf.METASTORE_MYSQL_PROPS);
+    if (mysqlProps == null) {
+      throw new Exception(FBHiveConf.METASTORE_MYSQL_PROPS + " is not defined");
+    }
+    if (urlFactory == null) {
+      urlFactory = HookUtils.getUrlFactory(
+        conf,
+        FBHiveConf.CONNECTION_FACTORY,
+        null,
+        FBHiveConf.METASTORE_MYSQL_TIER_VAR_NAME,
+        null,
+        FBHiveConf.METASTORE_MYSQL_PROPS);
+    }
+
+    urlFactory.updateProps(smcUrl, mysqlTier, mysqlProps);
+    return urlFactory.getUrl();
+  }
+
+  @Override
+  public void notifyBadConnectionUrl(String url) {
+    LOG.error("Notified of a bad URL: " + url);
+  }
+
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/metastore/hooks/StatsManager.java Tue May  1 22:52:38 2012
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import java.util.Map;
+
+
+public interface StatsManager {
+  public boolean containsKey(String statName);
+  public void addCountStatType(String statName);
+  public void addStatValue(String statName, int value);
+  public Map<String, Long> getCounters();
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AbstractSmcConfigHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.thrift.TException;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+/**
+ * An abstract class which should be extended by hooks which read configurations from an SMC
+ * config tier.
+ */
+public abstract class AbstractSmcConfigHook {
+
+  static final private Log LOG = LogFactory.getLog(AbstractSmcConfigHook.class);
+
+  private static final String CONFIG_FIELD = "config";
+  private static final String ENABLED_FIELD = "enabled";
+  private ThreadLocal<ConnectionUrlFactory> urlFactory = null;
+
+  /**
+   * Given a HiveConf, checks if the SMC hook enabled config is set to true
+   *
+   * @param conf
+   * @return
+   */
+  protected boolean isEnabled(HiveConf conf) {
+    boolean enabled = conf.getBoolean(FBHiveConf.ENABLED_CONFIG, false);
+
+    if (!enabled) {
+      LOG.error("SMC hook is not enabled.");
+    }
+
+    return enabled;
+  }
+
+  /**
+   * In each top level config object (jo) there is an enabled field.  This method checks that that
+   * field exists, is set properly, and is set to true.
+   *
+   * @param jo
+   * @param packageName
+   * @return
+   * @throws JSONException
+   */
+  protected boolean isConfigEnabled(JSONObject jo, String packageName) throws JSONException {
+    boolean enabled = false;
+
+    Object enabledObj = null;
+
+    if (jo.has(ENABLED_FIELD)) {
+      enabledObj = jo.get(ENABLED_FIELD);
+    }
+
+    if (enabledObj == null || !(enabledObj instanceof Boolean) ) {
+      LOG.error("enabled not properly set!");
+      return false;
+    }
+
+    enabled = enabledObj.equals(Boolean.TRUE);
+
+    if (!enabled) {
+      LOG.error("package " + packageName + " is not enabled");
+    }
+
+    return enabled;
+  }
+
+  /**
+   * Given a HiveConf object, this method goes to the config tier and retrieves the underlying
+   * config object (whether that's an array, object, or any other type of JSON).  It also performs
+   * checks that the tier can be retrieved, the package name is set, the config is enabled, etc.
+   *
+   * @param conf
+   * @return
+   * @throws JSONException
+   * @throws ServiceException
+   * @throws TException
+   */
+  protected Object getConfigObject(HiveConf conf)
+      throws JSONException, Exception, TException {
+
+    // Get the properties for this package
+    String packageName = conf.get(FBHiveConf.FB_CURRENT_CLUSTER);
+    if (packageName == null) {
+      LOG.error("Unable to use configs stored in SMC - no hive package set.");
+      return null;
+    }
+
+    if (urlFactory == null) {
+      urlFactory = new ThreadLocal<ConnectionUrlFactory>();
+      urlFactory.set(HookUtils.getUrlFactory(conf,
+                                             FBHiveConf.CONNECTION_FACTORY, null, null, null));
+    }
+
+    String s = urlFactory.get().getValue(conf.get(FBHiveConf.HIVE_CONFIG_TIER), packageName);
+    JSONObject jo = new JSONObject(s);
+
+    Object configObj = null;
+
+    if (!isConfigEnabled(jo, packageName)) {
+      return null;
+    }
+
+    if (jo.has(CONFIG_FIELD)) {
+      configObj = jo.get(CONFIG_FIELD);
+    }
+
+    return configObj;
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AlterTableRestrictHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Adding a hook to stop platinum tables from getting modified in silver
+ */
+public class AlterTableRestrictHook implements ExecuteWithHookContext {
+  static final private Log LOG = LogFactory.getLog("hive.ql.hooks.AlterTableRestrictHook");
+
+  String current_cluster = null;
+
+  public AlterTableRestrictHook() throws Exception {
+    HiveConf conf = new HiveConf(AlterTableRestrictHook.class);
+    current_cluster = conf.get(FBHiveConf.FB_CURRENT_CLUSTER);
+  }
+
+  /**
+   * Restrict the alter table command if the current cluster is not the same
+   * as the creation cluster
+   *
+   */
+  public void run(HookContext hookContext) throws Exception {
+    SessionState ss = SessionState.get();
+
+    if ((current_cluster == null) || (ss == null)) {
+      return;
+    }
+
+    HiveOperation commandType = ss.getHiveOperation();
+
+    // This check is only for alter table
+    if (!((commandType == HiveOperation.ALTERTABLE_ADDCOLS) ||
+          (commandType == HiveOperation.ALTERTABLE_REPLACECOLS) ||
+          (commandType == HiveOperation.ALTERTABLE_RENAMECOL) ||
+          (commandType == HiveOperation.ALTERTABLE_RENAMEPART) ||
+          (commandType == HiveOperation.ALTERTABLE_RENAME) ||
+          (commandType == HiveOperation.ALTERTABLE_PROPERTIES) ||
+          (commandType == HiveOperation.ALTERTABLE_SERIALIZER) ||
+          (commandType == HiveOperation.ALTERTABLE_SERDEPROPERTIES) ||
+          (commandType == HiveOperation.ALTERTABLE_CLUSTER_SORT) ||
+          (commandType == HiveOperation.ALTERTABLE_FILEFORMAT))) {
+      return;
+    }
+
+    // If the creation cluster is being modified to be the current cluster the alter should not be
+    // restricted
+    if (commandType == HiveOperation.ALTERTABLE_PROPERTIES) {
+      Map<String, String> newProps =
+        ((DDLWork)(hookContext.getQueryPlan().getRootTasks().get(0).getWork()))
+        .getAlterTblDesc().getProps();
+      if (newProps.containsKey(HookUtils.TABLE_CREATION_CLUSTER) &&
+          (newProps.get(HookUtils.TABLE_CREATION_CLUSTER).equals(current_cluster))) {
+        return;
+      }
+    }
+
+    Set<WriteEntity> outputs = hookContext.getOutputs();
+    for (WriteEntity output : outputs) {
+      Table table = output.getT();
+      if (table != null) {
+        String tableCreationCluster = table.getProperty(HookUtils.TABLE_CREATION_CLUSTER);
+        if (tableCreationCluster != null &&
+            !tableCreationCluster.isEmpty() &&
+            !tableCreationCluster.equals(current_cluster)) {
+          String exception = "Table " + table.getTableName() + " cannot be modified.";
+          exception += " Table's cluster is " + tableCreationCluster + ",";
+          exception += "whereas current package is " + current_cluster;
+          throw new Exception(exception);
+        }
+      }
+    }
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ArchiverHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Implementation of a pre execute hook that checks whether
+ * a partition is archived or not and also sets that query
+ * time for the partition.
+ */
+public class ArchiverHook implements PreExecute {
+
+  private static final String ARCHIVE_FLAG = "archiveFlag";
+  private static final String LAST_QUERY_TIME = "lastQueryTime";
+
+  static final private Log LOG = LogFactory.getLog("hive.ql.hooks.ArchiverHook");
+
+  /**
+   * The metastore client.
+   */
+  private HiveMetaStoreClient ms;
+
+  /**
+   * The archiver hook constructor.
+   */
+  public ArchiverHook() throws Exception {
+    ms = new HiveMetaStoreClient(new HiveConf(this.getClass()));
+  }
+
+  private Map<String, String> modifyParams(Map<String, String> old_map, String key, String value) {
+    Map<String, String> new_map = old_map;
+    if (new_map == null)
+      new_map = new LinkedHashMap<String, String>();
+    new_map.put(key, value);
+    return new_map;
+  }
+
+  private boolean setLastQueryTime(Table t) throws Exception {
+    Map<String, String> old_map = t.getParameters();
+    if (old_map != null) {
+      String timeStr = old_map.get(LAST_QUERY_TIME);
+      if (timeStr != null) {
+        long time = Long.parseLong(timeStr);
+        long cur_time = System.currentTimeMillis();
+        if (cur_time - time < 1*60*60*1000) {
+          // lastQueryTime was recently set
+          return false;
+        }
+      }
+    }
+    t.setParameters(modifyParams(old_map, LAST_QUERY_TIME,
+                    Long.toString(System.currentTimeMillis())));
+    return true;
+  }
+
+  private boolean setArchiveFlag(Table t) {
+    Map<String, String> old_map = t.getParameters();
+    if (old_map != null) {
+      String archF = old_map.get(ARCHIVE_FLAG);
+      if (archF != null) {
+        if(archF.equals("false")) {
+          return false;
+        }
+      }
+    }
+    t.setParameters(modifyParams(t.getParameters(), ARCHIVE_FLAG, "false"));
+    return true;
+  }
+
+  private boolean setLastQueryTime(Partition p) throws Exception {
+    Map<String, String> old_map = p.getParameters();
+    if (old_map != null) {
+      String timeStr = old_map.get(LAST_QUERY_TIME);
+      if (timeStr != null) {
+        long time = Long.parseLong(timeStr);
+        long cur_time = System.currentTimeMillis();
+        if (cur_time - time < 1*60*60*1000) {
+          // lastQueryTime was recently set
+          return false;
+        }
+      }
+    }
+    p.setParameters(modifyParams(old_map, LAST_QUERY_TIME,
+                    Long.toString(System.currentTimeMillis())));
+    return true;
+  }
+
+  private boolean setArchiveFlag(Partition p) {
+    Map<String, String> old_map = p.getParameters();
+    if (old_map != null) {
+      String archF = old_map.get(ARCHIVE_FLAG);
+      if (archF != null) {
+        if(archF.equals("false")) {
+          return false;
+        }
+      }
+    }
+    p.setParameters(modifyParams(p.getParameters(), ARCHIVE_FLAG, "false"));
+    return true;
+  }
+
+  public void run(SessionState sess, Set<ReadEntity> inputs,
+      Set<WriteEntity> outputs, UserGroupInformation ugi)
+    throws Exception {
+
+    //Go over the input paths and check if they are archived or not
+    for(ReadEntity re: inputs) {
+      boolean isArchived = false;
+      if (re.getParameters() != null) {
+        String archF = re.getParameters().get(ARCHIVE_FLAG);
+        if (archF != null) {
+          isArchived = archF.equals("true");
+        }
+      }
+
+      if (isArchived)
+        throw new Exception("Path: " + re.getLocation().toString() + " needs to be unarchived.");
+
+      // Set the last query time
+      ReadEntity.Type typ = re.getType();
+      switch(typ) {
+      case TABLE:
+        Table t = re.getTable().getTTable();
+        if(setLastQueryTime(t)) {
+          LOG.debug("Setting LastQueryTime for table " + re);
+          ms.alter_table(MetaStoreUtils.DEFAULT_DATABASE_NAME, t.getTableName(), t);
+        }
+        break;
+      case PARTITION:
+        Partition p = re.getPartition().getTPartition();
+        if (setLastQueryTime(p)) {
+          LOG.debug("Setting LastQueryTime for partition " + re);
+          ms.alter_partition(MetaStoreUtils.DEFAULT_DATABASE_NAME, p.getTableName(), p);
+        }
+        break;
+      default:
+        throw new Exception("Unknown type for input: " + re.toString());
+      }
+    }
+
+    // Go over the write paths and set the archived flag to false
+    for(WriteEntity we: outputs) {
+      WriteEntity.Type typ = we.getType();
+      boolean q, a;
+
+      switch(typ) {
+      case TABLE:
+        Table t = we.getTable().getTTable();
+        q = setLastQueryTime(t);
+        a = setArchiveFlag(t);
+        if(q || a) {
+          LOG.debug("Altering dest table for archiver " + we);
+          ms.alter_table(MetaStoreUtils.DEFAULT_DATABASE_NAME, t.getTableName(), t);
+        }
+        break;
+      case PARTITION:
+        Partition p = we.getPartition().getTPartition();
+        q = setLastQueryTime(p);
+        a = setArchiveFlag(p);
+        if(q || a) {
+          if (ms.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+                              p.getTableName(), p.getValues()) != null) {
+            LOG.debug("Altering dest partition for archiver " + we);
+            ms.alter_partition(MetaStoreUtils.DEFAULT_DATABASE_NAME, p.getTableName(), p);
+          }
+        }
+        break;
+      case DFS_DIR:
+      case LOCAL_DIR:
+        break;
+      default:
+        throw new Exception("Unknown type for output: " + we.toString());
+      }
+    }
+  }
+
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditJoinHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskRunner;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Implementation of a post execute hook that checks whether a partition is
+ * archived or not and also sets that query time for the partition.
+ */
+public class AuditJoinHook implements ExecuteWithHookContext {
+  static final private Log LOG = LogFactory
+      .getLog("hive.ql.hooks.AuditJoinHook");
+
+  ConnectionUrlFactory urlFactory = null;
+
+  public AuditJoinHook() throws Exception {
+    HiveConf conf = new HiveConf(AuditJoinHook.class);
+    urlFactory = HookUtils.getUrlFactory(conf,
+      FBHiveConf.CONNECTION_FACTORY,
+      FBHiveConf.AUDIT_CONNECTION_FACTORY,
+      FBHiveConf.AUDIT_MYSQL_TIER_VAR_NAME,
+      FBHiveConf.AUDIT_HOST_DATABASE_VAR_NAME);
+  }
+
+  public void run(HookContext hookContext) throws Exception {
+    HiveConf conf = hookContext.getConf();
+    boolean enableConvert = HiveConf.getBoolVar(conf,
+        HiveConf.ConfVars.HIVECONVERTJOIN);
+    if (!enableConvert) {
+      return;
+    }
+    String command = StringEscapeUtils.escapeJava(SessionState.get()
+        .getCmd());
+    QueryPlan plan = hookContext.getQueryPlan();
+    String queryID = StringEscapeUtils.escapeJava(plan.getQueryId());
+    // String query = SessionState.get().getCmd();
+
+    int convertedMapJoin = 0;
+    int commonJoin = 0;
+    int backupCommonJoin = 0;
+    int convertedLocalMapJoin = 0;
+    int localMapJoin = 0;
+
+    List<TaskRunner> list = hookContext.getCompleteTaskList();
+    for (TaskRunner tskRunner : list) {
+      Task tsk = tskRunner.getTask();
+      int tag = tsk.getTaskTag();
+      switch (tag) {
+      case Task.COMMON_JOIN:
+        commonJoin++;
+        break;
+      case Task.CONVERTED_LOCAL_MAPJOIN:
+        convertedLocalMapJoin++;
+        break;
+      case Task.CONVERTED_MAPJOIN:
+        convertedMapJoin++;
+        break;
+      case Task.BACKUP_COMMON_JOIN:
+        backupCommonJoin++;
+        break;
+      case Task.LOCAL_MAPJOIN:
+        localMapJoin++;
+        break;
+      }
+    }
+
+    // nothing to do
+    if ((convertedMapJoin == 0) &&
+         (commonJoin == 0) &&
+         (backupCommonJoin == 0) &&
+         (convertedLocalMapJoin == 0)
+        && (localMapJoin == 0)) {
+      return;
+    }
+
+    ArrayList<Object> sqlParams = new ArrayList<Object>();
+    sqlParams.add(StringEscapeUtils.escapeJava(command));
+    sqlParams.add(StringEscapeUtils.escapeJava(queryID));
+    sqlParams.add(new Integer(convertedLocalMapJoin));
+    sqlParams.add(new Integer(convertedMapJoin));
+    sqlParams.add(new Integer(localMapJoin));
+    sqlParams.add(new Integer(commonJoin));
+    sqlParams.add(new Integer(backupCommonJoin));
+
+    String sql = "insert into audit_join set " +
+        " command = ?, query_id = ?, converted_local_mapjoin = ?, converted_map_join = ?," +
+        " local_mapjoin = ?, common_join = ?, backup_common_join = ?";
+
+    if (urlFactory == null) {
+      urlFactory = HookUtils.getUrlFactory(
+          conf,
+          FBHiveConf.CONNECTION_FACTORY,
+          FBHiveConf.AUDIT_CONNECTION_FACTORY,
+          FBHiveConf.AUDIT_MYSQL_TIER_VAR_NAME,
+          FBHiveConf.AUDIT_HOST_DATABASE_VAR_NAME);
+      if (urlFactory == null) {
+        throw new RuntimeException("DB parameters not set!");
+      }
+    }
+
+    HookUtils.runInsert(conf,
+                        urlFactory, sql, sqlParams, HookUtils
+                        .getSqlNumRetry(conf));
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/AuditLocalModeHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskRunner;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+/**
+ * Implementation of a post execute hook that checks whether a partition is
+ * archived or not and also sets that query time for the partition.
+ */
+public class AuditLocalModeHook implements ExecuteWithHookContext {
+  static final private Log LOG = LogFactory
+      .getLog("hive.ql.hooks.AuditLocalModeHook");
+  final static String MYSQL_TIER_VAR_NAME = "fbhive.audit.mysql.tier";
+  final static String HOST_DATABASE_VAR_NAME = "fbhive.audit.mysql";
+
+  ConnectionUrlFactory urlFactory = null;
+
+  public AuditLocalModeHook() throws Exception {
+    HiveConf conf = new HiveConf(AuditLocalModeHook.class);
+    urlFactory = HookUtils.getUrlFactory(conf,
+        FBHiveConf.CONNECTION_FACTORY,
+        FBHiveConf.AUDIT_CONNECTION_FACTORY,
+        FBHiveConf.AUDIT_MYSQL_TIER_VAR_NAME,
+        FBHiveConf.AUDIT_HOST_DATABASE_VAR_NAME);
+  }
+
+  public void run(HookContext hookContext) throws Exception {
+    HiveConf conf = hookContext.getConf();
+    String command = StringEscapeUtils.escapeJava(SessionState.get().getCmd());
+    QueryPlan plan = hookContext.getQueryPlan();
+    String queryID = StringEscapeUtils.escapeJava(plan.getQueryId());
+    int numLocalModeTask = 0;
+    int numMapRedTask = 0;
+    int numTask = 0;
+    List<TaskRunner> list = hookContext.getCompleteTaskList();
+    numTask = list.size();
+    for (TaskRunner tskRunner : list) {
+      Task tsk = tskRunner.getTask();
+      if(tsk.isMapRedTask()){
+        if(tsk.isLocalMode()) {
+            numLocalModeTask++;
+           }
+        numMapRedTask++;
+      }
+    }
+    if(numLocalModeTask == 0){
+      return ;
+    }
+    ArrayList<Object> sqlParams = new ArrayList<Object>();
+    sqlParams.add(StringEscapeUtils.escapeJava(command));
+    sqlParams.add(StringEscapeUtils.escapeJava(queryID));
+    sqlParams.add(new Integer(numTask));
+    sqlParams.add(new Integer(numMapRedTask));
+    sqlParams.add(new Integer(numLocalModeTask));
+
+    String sql = "insert into audit_local set " +
+                  " command = ?, query_id = ?, num_tasks = ?, num_mapred_tasks = ?," +
+    " num_local_mapred_tasks = ?";
+    HookUtils.runInsert(conf, urlFactory, sql, sqlParams, HookUtils
+                        .getSqlNumRetry(conf));
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/BaseReplicationHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+
+/**
+ * Seperating out some functionality for so that Hive1 can share code.
+ */
+public class BaseReplicationHook {
+  static final private Log LOG = LogFactory.getLog("hive.ql.hooks.BaseReplicationHook");
+
+  protected ConnectionUrlFactory urlFactory = null;
+  HiveConf conf = null;
+
+  public static ConnectionUrlFactory getReplicationMySqlUrl() {
+    HiveConf conf = new HiveConf(BaseReplicationHook.class);
+    return HookUtils.getUrlFactory(conf,
+        FBHiveConf.CONNECTION_FACTORY,
+        FBHiveConf.REPLICATION_CONNECTION_FACTORY,
+        FBHiveConf.REPLICATION_MYSQL_TIER_VAR_NAME,
+        FBHiveConf.REPLICATION_HOST_DATABASE_VAR_NAME);
+  }
+
+  public BaseReplicationHook() throws Exception {
+    urlFactory = getReplicationMySqlUrl();
+    conf = new HiveConf(BaseReplicationHook.class);
+  }
+
+  /**
+   * Simplified call used by hive1 to insert into the audit log
+   *
+   * @param command
+   * @param commandType
+   * @param inputs
+   * @param outputs
+   * @param userInfo
+   * @throws Exception
+   */
+  public void run(String command, String commandType, String inputs,
+      String outputs, String userInfo) throws Exception {
+    ArrayList<Object> sqlParams = new ArrayList<Object>();
+    sqlParams.add(StringEscapeUtils.escapeJava(commandType));
+    sqlParams.add(StringEscapeUtils.escapeJava(inputs));
+    sqlParams.add(outputs);
+    sqlParams.add(StringEscapeUtils.escapeJava(userInfo));
+
+    String sql = "insert into snc1_command_log set command_type = ?, " +
+      "inputs = ?, outputs = ?, user_info = ?";
+    if (conf == null) {
+      conf = new HiveConf(BaseReplicationHook.class);
+    }
+    HookUtils.runInsert(conf, urlFactory, sql, sqlParams);
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckArchivedDataHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.Set;
+
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+/**
+ * Implementation of a pre execute hook that checks whether
+ * a partition is archived or not
+ */
+public class CheckArchivedDataHook {
+
+  private static final String ARCHIVE_FLAG = "archivedFlag";
+  final static String DISABLE_CHECK_ARCHIVAL_HOOK = "fbhive.disable.checkArchival.hook";
+
+  public static class PreExec implements PreExecute {
+
+    public void run(SessionState sess, Set<ReadEntity> inputs,
+                    Set<WriteEntity> outputs, UserGroupInformation ugi)
+      throws Exception {
+
+      // Did the user explicitly ask to disable the hook
+      HiveConf conf = sess.getConf();
+      String   disableArch = conf.get(DISABLE_CHECK_ARCHIVAL_HOOK);
+      if ((disableArch != null) && (disableArch.compareToIgnoreCase("false") == 0)) {
+        return;
+      }
+
+      //Go over the input paths and check if they are archived or not
+      for(ReadEntity re: inputs) {
+        boolean isArchived = false;
+        if (re.getParameters() != null) {
+          String archF = re.getParameters().get(ARCHIVE_FLAG);
+          if (archF != null) {
+            isArchived = archF.equals("true");
+            if (isArchived)
+              throw new Exception("Path: " + re.getLocation().toString() +
+                                  " needs to be unarchived.");
+          }
+        }
+      }
+    }
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CheckRetentionsHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.DDLTask;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
+
+
+/**
+ * Implementation of a pre execute hook that checks the table RETENTION is set.
+ */
+public class CheckRetentionsHook {
+
+  private static final Log LOG = LogFactory.getLog(CheckRetentionsHook.class.getName());
+  private static HiveConf conf;
+
+  // If warningOnly = true, we print out some warnnings without fail
+  // the CREATE TABLE DDL.
+  private static boolean warningOnly = false;
+
+  // required table parameters
+  private static final String RETENTION_FLAG = "RETENTION";
+  private static final String RETENTION_PLATINUM_FLAG = "RETENTION_PLATINUM";
+
+  // wiki page URL that explains the policies
+  private static final String wikiURL =
+    "https://www.intern.facebook.com/intern/wiki/index.php/Data/Hive/" +
+    "Retention_on_new_tables";
+
+  private static String retentionKey = null;
+
+  private static String ErrMsg(String str) {
+    return str + "\n  Here's how to add retention: " + wikiURL;
+  }
+
+  public static class PreExec implements ExecuteWithHookContext {
+
+    public void run(HookContext hookContext)
+      throws Exception {
+
+      assert(hookContext.getHookType() == HookContext.HookType.PRE_EXEC_HOOK);
+
+      SessionState sess = SessionState.get();
+      Set<ReadEntity> inputs = hookContext.getInputs();
+      Set<WriteEntity> outputs = hookContext.getOutputs();
+      UserGroupInformation ugi = hookContext.getUgi();
+      conf = sess.getConf();
+
+      warningOnly = conf.getBoolean(FBHiveConf.NO_RETENTION_WARNING_ONLY, true);
+
+      // figure out if we are on silver or platinum
+      String whDir = HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE);
+      if (whDir == null) {
+        throw new Exception(ErrMsg("Cannot determine which cluster this query is running on: " +
+          "hive.metastore.warehouse.dir is not set!"));
+      }
+
+      Path p = new Path(whDir);
+      String hostName =  p.toUri().getHost();
+
+      if (hostName.equals(conf.get(FBHiveConf.FBHIVE_SILVER_DFS_PREFIX)) ||
+          hostName.equals(conf.get(FBHiveConf.FBHIVE_SILVER_DFS_PREFIX2)) ||
+          hostName.equals(conf.get(FBHiveConf.FBHIVE_SILVER_DFS_PREFIX3))) {
+        retentionKey = RETENTION_FLAG;
+      } else if (hostName.equals(conf.get(FBHiveConf.FBHIVE_PLATINUM_DFS_PREFIX))) {
+        retentionKey = RETENTION_PLATINUM_FLAG;
+      } else {
+        throw new Exception(ErrMsg("Cannot determine which cluster this query is running on: " +
+          "hive.metastore.warehouse.dir=" + whDir +
+         "; does not seems to belong to either silver or platinum!"));
+      }
+
+      Set<Task<? extends Serializable>> tasks = new HashSet<Task<? extends Serializable>>();
+      getReachableTasks(tasks, hookContext.getQueryPlan());
+
+      for (Task task: tasks) {
+        if (task instanceof DDLTask) {
+          DDLWork work = (DDLWork) task.getWork();
+          if (work.getCreateTblDesc() != null) {
+            checkRetention(work.getCreateTblDesc(), retentionKey);
+          }
+        }
+      }
+    }
+
+    private void getReachableTasks(Set<Task<? extends Serializable>> tasks, QueryPlan qp) {
+      ArrayList<Task<? extends Serializable>> rootTasks = qp.getRootTasks();
+      for (Task<? extends Serializable> task: rootTasks) {
+        getReachableTasks(tasks, task);
+      }
+    }
+
+    /**
+     * Recursively traverse the task dependence tree and gather all tasks into
+     * the set.
+     */
+    private void getReachableTasks(Set<Task<? extends Serializable>> tasks,
+            Task<? extends Serializable> rootTask) {
+      if (!tasks.contains(rootTask)) {
+        tasks.add(rootTask);
+        if (rootTask.getDependentTasks() != null) {
+          for (Task<? extends Serializable> child: rootTask.getDependentTasks()) {
+            getReachableTasks(tasks, child);
+          }
+        }
+      }
+    }
+
+    private void warnOrFail(boolean warning, String mesg) throws Exception {
+      if (warning) {
+        // shout loud on stderr!
+        System.err.println("\n ----------");
+        System.err.println("| WARNING: | ");
+        System.err.println(" ----------");
+        System.err.println("  This command does NOT comply with the RETENTION " +
+            "policies. This command will fail in the near future. \n" +
+            mesg);
+      } else {
+        throw new Exception(mesg);
+      }
+    }
+
+
+    /**
+     * Check if the CREATE TABLE statement has retention and data growth
+     * estimation set. If not throw an exception.
+     */
+    private void checkRetention(CreateTableDesc desc, String retentionKey)
+        throws Exception {
+
+      // exclude EXTERNAL tables
+      if (desc.isExternal()) {
+        return;
+      }
+
+      // TODO: remove this whenever it becomes feasible
+      // exclude table name starts with tmp, temp, or test: tmp tables should be set
+      // default retention
+      if (desc.getTableName().startsWith("tmp") ||
+          desc.getTableName().startsWith("temp") ||
+          desc.getTableName().startsWith("test")) {
+        return;
+      }
+
+      // check if table already exists
+      if (tableExists(desc.getTableName())) {
+        return;
+      }
+
+      String tableNeedsRetention = "Newly created tables have to have " + retentionKey +
+              " set unless the table name has one of the prefixes \"tmp\", \"temp\", or \"test\".";
+
+      String tableRetentionFormat = "The value of the " + retentionKey + " parameter must be an " +
+              "integer greater than or equal to -1, i.e. -1,0,1,...";
+
+      // check 'RETENTION' parameter exists
+      String retentionValue = "";
+      if (desc.getTblProps() == null ||
+          (retentionValue = desc.getTblProps().get(retentionKey)) == null) {
+        warnOrFail(warningOnly, ErrMsg("Table " + desc.getTableName() + " does not have " +
+                retentionKey + " parameter set. "
+                + tableNeedsRetention + "  " + tableRetentionFormat));
+        return;
+      }
+
+      // check 'RETENTION' parameter is set to a value in the range -1,0,1,...
+      int retentionIntValue;
+      try {
+        retentionIntValue = Integer.parseInt(retentionValue);
+      } catch (Exception e) {
+        // retentionValue is not a valid integer, set retentionIntValue to an invalid value
+        retentionIntValue = Integer.MIN_VALUE;
+      }
+
+      if (retentionIntValue < -1) {
+        warnOrFail(warningOnly, ErrMsg("Table " + desc.getTableName() + " has an invalid value " +
+                retentionValue + " for the parameter " + retentionKey + ".  " +
+                tableRetentionFormat + "  " + tableNeedsRetention));
+      }
+    }
+
+    private boolean tableExists(String tabName) throws Exception {
+      Hive db = Hive.get(conf);
+      Table table = db.getTable("default", tabName, false);
+      return table != null;
+    }
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConfUrlFactory.java Tue May  1 22:52:38 2012
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ *
+ * This factory creates the connection URL from the supplied configuration.
+ *
+ */
+
+public class ConfUrlFactory implements ConnectionUrlFactory {
+
+  HiveConf conf = null;
+  String confVarName = null;
+  public ConfUrlFactory() {
+    this(new HiveConf(ConfUrlFactory.class), "");
+  }
+
+  public ConfUrlFactory(String confVarName) {
+    this(new HiveConf(ConfUrlFactory.class), confVarName);
+  }
+
+  public ConfUrlFactory(HiveConf conf, String confVarName) {
+    this.conf = conf;
+    this.confVarName = confVarName;
+  }
+
+  public boolean init(Configuration hconf) {
+    this.conf = (HiveConf)hconf;
+    return true;
+  }
+
+  @Override
+  public void init(String param1Name, String param2Name) {
+    this.confVarName = param1Name;
+  }
+
+  @Override
+  public String getUrl() throws Exception {
+    String dbstr = conf.get(confVarName);
+    String[] hostDatabases = dbstr.split(":");
+    return "jdbc:mysql://" + hostDatabases[0] + "/" + hostDatabases[1];
+  }
+
+  @Override
+  public String getUrl(boolean isWrite) throws Exception {
+    return getUrl();
+  }
+
+  @Override
+  public String getValue(String param1, String param2) throws Exception {
+    return null;
+  }
+
+  @Override
+  public void updateProps(String param1, String param2, String param3) {
+    return;
+  }
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/ConnectionUrlFactory.java Tue May  1 22:52:38 2012
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+/**
+ *
+ * Classes implementing this interface create JDBC connection URL's.
+ * This can also be used to store a parameters array
+ */
+public interface ConnectionUrlFactory {
+
+  public void init(String param1Name, String param2Name);
+
+  /**
+   * @return the JDBC connection URL
+   * @throws Exception
+   */
+  String getUrl() throws Exception;
+
+  String getUrl(boolean isWrite) throws Exception;
+
+  String getValue(String param1, String param2) throws Exception;
+  void updateProps(String param1, String param2, String param3);
+}

Added: hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java?rev=1332873&view=auto
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java (added)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/ql/hooks/CreateTableChangeDFSHook.java Tue May  1 22:52:38 2012
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.hooks.conf.FBHiveConf;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
+import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+
+/**
+ * Implementation of a pre execute hook that is used to change
+ * the location of the DFS.
+ * This is only applicable to new tables - this can be used to
+ * eventually spread the load evenly on more than 1 DFS.
+ */
+public class CreateTableChangeDFSHook implements ExecuteWithHookContext {
+  static final private Log LOG = LogFactory.getLog(CreateTableChangeDFSHook.class.getName());
+
+  public void run(HookContext hookContext) throws Exception {
+    assert(hookContext.getHookType() == HookContext.HookType.PRE_EXEC_HOOK);
+
+    QueryPlan queryPlan = hookContext.getQueryPlan();
+
+    // This change is only needed when a new table is being created
+    ArrayList<Task<? extends Serializable>> rootTasks = queryPlan.getRootTasks();
+
+    if ((rootTasks == null) || (rootTasks.size() != 1)) {
+      return;
+    }
+
+    Task<? extends Serializable> tsk = rootTasks.get(0);
+
+    if (!(tsk.getWork() instanceof DDLWork)) {
+      return;
+    }
+
+    HiveConf conf = hookContext.getConf();
+    DDLWork ddlWork = (DDLWork)tsk.getWork();
+
+    float pubPercent = conf.getFloat(FBHiveConf.ENABLE_PARTIAL_CHANGEDFS, 0);
+
+    // if pubPercent == 0, make sure it returns.
+    if (!HookUtils.rollDice(pubPercent)) {
+      return;
+    }
+
+    String newDir = conf.get(FBHiveConf.SECONDARYMETASTOREWAREHOUSE);
+
+    if (ddlWork.getCreateTblDesc() != null) {
+      CreateTableDesc crtTblDesc = ddlWork.getCreateTblDesc();
+      // The user has already specified the location
+      if (crtTblDesc.getLocation() != null) {
+        return;
+      }
+
+      // This is only for tmp tables right now
+      if ((crtTblDesc.getTableName() == null) ||
+          ((!crtTblDesc.getTableName().startsWith("tmp_")) &&
+           (!crtTblDesc.getTableName().startsWith("temp_")))) {
+        return;
+      }
+
+      String locn =
+        (new Warehouse(conf)).getTablePath(newDir, crtTblDesc.getTableName()).toString();
+      crtTblDesc.setLocation(locn);
+      LOG.info("change location for table " + crtTblDesc.getTableName());
+      return;
+    }
+
+    if (ddlWork.getCreateTblLikeDesc() != null) {
+      CreateTableLikeDesc crtTblLikeDesc = ddlWork.getCreateTblLikeDesc();
+      // The user has already specified the location
+      if (crtTblLikeDesc.getLocation() != null) {
+        return;
+      }
+
+      // This is only for tmp tables right now
+      if ((crtTblLikeDesc.getTableName() == null) ||
+          ((!crtTblLikeDesc.getTableName().startsWith("tmp_")) &&
+           (!crtTblLikeDesc.getTableName().startsWith("temp_")))) {
+        return;
+      }
+
+      String locn =
+        (new Warehouse(conf)).getTablePath(newDir, crtTblLikeDesc.getTableName()).toString();
+      crtTblLikeDesc.setLocation(locn);
+      LOG.info("change location for table " + crtTblLikeDesc.getTableName());
+      return;
+    }
+  }
+}



Mime
View raw message