hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1665403 [2/7] - in /hive/branches/spark: ./ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/ an...
Date Tue, 10 Mar 2015 04:20:48 GMT
Modified: hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java (original)
+++ hive/branches/spark/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java Tue Mar 10 04:20:45 2015
@@ -20,12 +20,15 @@ package org.apache.hive.hcatalog.api;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.ObjectPair;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
 import org.apache.hadoop.hive.common.classification.InterfaceStability;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -36,6 +39,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
 import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -48,12 +52,25 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.UnknownDBException;
 import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hive.hcatalog.common.HCatConstants;
 import org.apache.hive.hcatalog.common.HCatException;
 import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils;
 import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The HCatClientHMSImpl is the Hive Metastore client based implementation of
@@ -61,6 +78,7 @@ import org.apache.thrift.TException;
  */
 public class HCatClientHMSImpl extends HCatClient {
 
+  private static final Logger LOG = LoggerFactory.getLogger(HCatClientHMSImpl.class);
   private HiveMetaStoreClient hmsClient;
   private Configuration config;
   private HiveConf hiveConfig;
@@ -480,19 +498,122 @@ public class HCatClientHMSImpl extends H
     }
   }
 
+  /**
+   * Helper class to help build ExprDesc tree to represent the partitions to be dropped.
+   * Note: At present, the ExpressionBuilder only constructs partition predicates where
+   * partition-keys equal specific values, and logical-AND expressions. E.g.
+   *  ( dt = '20150310' AND region = 'US' )
+   * This only supports the partition-specs specified by the Map argument of:
+   * {@link org.apache.hive.hcatalog.api.HCatClient#dropPartitions(String, String, Map, boolean)}
+   */
+  private static class ExpressionBuilder {
+
+    private Map<String, PrimitiveTypeInfo> partColumnTypesMap = Maps.newHashMap();
+    private Map<String, String> partSpecs;
+
+    public ExpressionBuilder(Table table, Map<String, String> partSpecs) {
+      this.partSpecs = partSpecs;
+      for (FieldSchema partField : table.getPartitionKeys()) {
+        partColumnTypesMap.put(partField.getName().toLowerCase(),
+            TypeInfoFactory.getPrimitiveTypeInfo(partField.getType()));
+      }
+    }
+
+    private PrimitiveTypeInfo getTypeFor(String partColumn) {
+      return partColumnTypesMap.get(partColumn.toLowerCase());
+    }
+
+    private Object getTypeAppropriateValueFor(PrimitiveTypeInfo type, String value) {
+      ObjectInspectorConverters.Converter converter = ObjectInspectorConverters.getConverter(
+          TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(TypeInfoFactory.stringTypeInfo),
+          TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(type));
+
+      return converter.convert(value);
+    }
+
+    public ExprNodeGenericFuncDesc equalityPredicate(String partColumn, String value) throws SemanticException {
+
+      PrimitiveTypeInfo partColumnType = getTypeFor(partColumn);
+      ExprNodeColumnDesc partColumnExpr = new ExprNodeColumnDesc(partColumnType, partColumn, null, true);
+      ExprNodeConstantDesc valueExpr = new ExprNodeConstantDesc(partColumnType,
+          getTypeAppropriateValueFor(partColumnType, value));
+
+      return binaryPredicate("=", partColumnExpr, valueExpr);
+    }
+
+    public ExprNodeGenericFuncDesc binaryPredicate(String function, ExprNodeDesc lhs, ExprNodeDesc rhs) throws SemanticException {
+      return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
+          FunctionRegistry.getFunctionInfo(function).getGenericUDF(),
+          Lists.newArrayList(lhs, rhs));
+    }
+
+    public ExprNodeGenericFuncDesc build() throws SemanticException {
+      ExprNodeGenericFuncDesc resultExpr = null;
+
+      for (Map.Entry<String,String> partSpec : partSpecs.entrySet()) {
+        String column = partSpec.getKey();
+        String value  = partSpec.getValue();
+        ExprNodeGenericFuncDesc partExpr = equalityPredicate(column, value);
+
+        resultExpr = (resultExpr == null? partExpr : binaryPredicate("and", resultExpr, partExpr));
+      }
+
+      return resultExpr;
+    }
+  } // class ExpressionBuilder;
+
+  private static boolean isExternal(Table table) {
+    return table.getParameters() != null
+        && "TRUE".equalsIgnoreCase(table.getParameters().get("EXTERNAL"));
+  }
+
+  private void dropPartitionsUsingExpressions(Table table, Map<String, String> partitionSpec, boolean ifExists)
+    throws SemanticException, TException {
+    LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions.");
+    ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build();
+    ObjectPair<Integer, byte[]> serializedPartitionExpression =
+        new ObjectPair<Integer, byte[]>(partitionSpec.size(),
+            Utilities.serializeExpressionToKryo(partitionExpression));
+    hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression),
+        !isExternal(table),  // Delete data?
+        false,               // Ignore Protection?
+        ifExists,            // Fail if table doesn't exist?
+        false);              // Need results back?
+  }
+
+  private void dropPartitionsIteratively(String dbName, String tableName,
+                                         Map<String, String> partitionSpec, boolean ifExists) throws HCatException, TException {
+    LOG.info("HCatClient: Dropping partitions iteratively.");
+    List<Partition> partitions = hmsClient.listPartitionsByFilter(dbName, tableName,
+        getFilterString(partitionSpec), (short) -1);
+    for (Partition partition : partitions) {
+      dropPartition(partition, ifExists);
+    }
+  }
+
   @Override
   public void dropPartitions(String dbName, String tableName,
                  Map<String, String> partitionSpec, boolean ifExists)
     throws HCatException {
+    LOG.info("HCatClient dropPartitions(db=" + dbName + ",table=" + tableName + ", partitionSpec: ["+ partitionSpec + "]).");
     try {
       dbName = checkDB(dbName);
-      List<Partition> partitions = hmsClient.listPartitionsByFilter(dbName, tableName,
-          getFilterString(partitionSpec), (short)-1);
+      Table table = hmsClient.getTable(dbName, tableName);
 
-      for (Partition partition : partitions) {
-        dropPartition(partition, ifExists);
+      if (hiveConfig.getBoolVar(HiveConf.ConfVars.METASTORE_CLIENT_DROP_PARTITIONS_WITH_EXPRESSIONS)) {
+        try {
+          dropPartitionsUsingExpressions(table, partitionSpec, ifExists);
+        }
+        catch (SemanticException parseFailure) {
+          LOG.warn("Could not push down partition-specification to back-end, for dropPartitions(). Resorting to iteration.",
+              parseFailure);
+          dropPartitionsIteratively(dbName, tableName, partitionSpec, ifExists);
+        }
+      }
+      else {
+        // Not using expressions.
+        dropPartitionsIteratively(dbName, tableName, partitionSpec, ifExists);
       }
-
     } catch (NoSuchObjectException e) {
       throw new ObjectNotFoundException(
           "NoSuchObjectException while dropping partition. " +

Modified: hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java (original)
+++ hive/branches/spark/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java Tue Mar 10 04:20:45 2015
@@ -26,6 +26,7 @@ import java.util.Map;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hive.hcatalog.templeton.JsonBuilder;
 
 /**
@@ -70,7 +71,7 @@ public class JobState {
     TempletonStorage storage = null;
     try {
       storage = (TempletonStorage)
-        Class.forName(conf.get(TempletonStorage.STORAGE_CLASS))
+          JavaUtils.loadClass(conf.get(TempletonStorage.STORAGE_CLASS))
           .newInstance();
     } catch (Exception e) {
       LOG.warn("No storage method found: " + e.getMessage());

Modified: hive/branches/spark/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java (original)
+++ hive/branches/spark/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java Tue Mar 10 04:20:45 2015
@@ -438,6 +438,41 @@ public class TestDbNotificationListener
   }
 
   @Test
+  public void sqlCTAS() throws Exception {
+
+    driver.run("create table ctas_source (c int)");
+    driver.run("insert into table ctas_source values (1)");
+    driver.run("create table ctas_target as select c from ctas_source");
+
+    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
+
+    assertEquals(6, rsp.getEventsSize());
+    NotificationEvent event = rsp.getEvents().get(0);
+    assertEquals(firstEventId + 1, event.getEventId());
+    assertEquals(HCatConstants.HCAT_CREATE_TABLE_EVENT, event.getEventType());
+    event = rsp.getEvents().get(2);
+    assertEquals(firstEventId + 3, event.getEventId());
+    assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
+    // Make sure the files are listed in the insert
+    assertTrue(event.getMessage().matches(".*\"files\":\\[\"pfile.*"));
+    event = rsp.getEvents().get(4);
+    assertEquals(firstEventId + 5, event.getEventId());
+    assertEquals(HCatConstants.HCAT_CREATE_TABLE_EVENT, event.getEventType());
+  }
+
+  @Test
+  public void sqlTempTable() throws Exception {
+
+    LOG.info("XXX Starting temp table");
+    driver.run("create temporary table tmp1 (c int)");
+    driver.run("insert into table tmp1 values (1)");
+
+    NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null);
+
+    assertEquals(0, rsp.getEventsSize());
+  }
+
+  @Test
   public void sqlDb() throws Exception {
 
     driver.run("create database sd");

Modified: hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java (original)
+++ hive/branches/spark/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java Tue Mar 10 04:20:45 2015
@@ -352,7 +352,7 @@ public class TestBeeLineWithArgs {
     argList.add("--outputformat=dsv");
     argList.add("--delimiterForDSV=;");
 
-    final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";1.0";
+    final String EXPECTED_PATTERN = "1;NULL;defg;ab\"c;1.0";
     testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
   }
 
@@ -365,7 +365,7 @@ public class TestBeeLineWithArgs {
     List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
     argList.add("--outputformat=tsv2");
 
-    final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t1.0";
+    final String EXPECTED_PATTERN = "1\tNULL\tdefg\tab\"c\t1.0";
     testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
   }
 
@@ -382,6 +382,81 @@ public class TestBeeLineWithArgs {
     testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
   }
 
+  /**
+   * Test writing output using new TSV format
+   */
+  @Test
+  public void testTSV2OutputWithDoubleQuotes() throws Throwable {
+    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
+    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+    argList.add("--outputformat=tsv2");
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV,"false");
+
+    final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t\"\"\"aa\"\"\"\t1.0";
+    testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
+  }
+
+  /**
+   * Test writing output using TSV deprecated format
+   */
+  @Test
+  public void testTSVOutputWithDoubleQuotes() throws Throwable {
+    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
+    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+    argList.add("--outputformat=tsv");
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");
+
+    final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c'\t'\"aa\"'\t'1.0'";
+    testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
+  }
+
+  /**
+   * Test writing output using new CSV format
+   */
+  @Test
+  public void testCSV2OutputWithDoubleQuotes() throws Throwable {
+    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
+    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+    argList.add("--outputformat=csv2");
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");
+
+    final String EXPECTED_PATTERN = "1,NULL,defg,\"ab\"\"c\",\"\"\"aa\"\"\",1.0";
+    testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
+  }
+
+  /**
+   * Test writing output using CSV deprecated format
+   */
+  @Test
+  public void testCSVOutputWithDoubleQuotes() throws Throwable {
+    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
+    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+    argList.add("--outputformat=csv");
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");
+
+    final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c','\"aa\"','1.0'";
+    testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
+  }
+
+  /**
+   * Test writing output using DSV format, with custom delimiter ";"
+   */
+  @Test
+  public void testDSVOutputWithDoubleQuotes() throws Throwable {
+    String SCRIPT_TEXT = getFormatTestQueryForEableQuotes();
+    List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+    argList.add("--outputformat=dsv");
+    argList.add("--delimiterForDSV=;");
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false");
+
+    final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";\"\"\"aa\"\"\";1.0";
+    testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+    System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true");
+  }
 
   /**
    * Test writing output using TSV deprecated format
@@ -428,6 +503,12 @@ public class TestBeeLineWithArgs {
     return "set hive.support.concurrency = false;\n" +
         "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n";
   }
+
+  private String getFormatTestQueryForEableQuotes() {
+    return "set hive.support.concurrency = false;\n" +
+        "select 1, null, 'defg', 'ab\"c', '\"aa\"', 1.0D from " + tableName + " limit 1 ;\n";
+  }
+
   /**
    * Select null from table , check if setting null to empty string works - Using beeling cmd line
    *  argument.

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Tue Mar 10 04:20:45 2015
@@ -220,6 +220,7 @@ import org.apache.thrift.transport.TTran
 
 import com.facebook.fb303.FacebookBase;
 import com.facebook.fb303.fb_status;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableListMultimap;
@@ -237,6 +238,12 @@ public class HiveMetaStore extends Thrif
   // embedded metastore or a remote one
   private static boolean isMetaStoreRemote = false;
 
+  // Used for testing to simulate method timeout.
+  @VisibleForTesting
+  static boolean TEST_TIMEOUT_ENABLED = false;
+  @VisibleForTesting
+  static long TEST_TIMEOUT_VALUE = -1;
+
   /** A fixed date format to be used for hive partition column values. */
   public static final ThreadLocal<DateFormat> PARTITION_DATE_FORMAT =
        new ThreadLocal<DateFormat>() {
@@ -470,6 +477,7 @@ public class HiveMetaStore extends Thrif
           hiveConf.getVar(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS));
       listeners = MetaStoreUtils.getMetaStoreListeners(MetaStoreEventListener.class, hiveConf,
           hiveConf.getVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS));
+      listeners.add(new SessionPropertiesListener(hiveConf));
       endFunctionListeners = MetaStoreUtils.getMetaStoreListeners(
           MetaStoreEndFunctionListener.class, hiveConf,
           hiveConf.getVar(HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS));
@@ -883,6 +891,15 @@ public class HiveMetaStore extends Thrif
           // expected
         }
 
+        if (TEST_TIMEOUT_ENABLED) {
+          try {
+            Thread.sleep(TEST_TIMEOUT_VALUE);
+          } catch (InterruptedException e) {
+            // do nothing
+          }
+          Deadline.checkTimeout();
+        }
+
         create_database_core(getMS(), db);
         success = true;
       } catch (Exception e) {
@@ -1518,8 +1535,16 @@ public class HiveMetaStore extends Thrif
         if (!success) {
           ms.rollbackTransaction();
         } else if (deleteData && !isExternal) {
-          boolean ifPurge = envContext != null &&
-              Boolean.parseBoolean(envContext.getProperties().get("ifPurge"));
+          // Data needs deletion. Check if trash may be skipped.
+          // Trash may be skipped iff:
+          //  1. deleteData == true, obviously.
+          //  2. tbl is external.
+          //  3. Either
+          //    3.1. User has specified PURGE from the commandline, and if not,
+          //    3.2. User has set the table to auto-purge.
+          boolean ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
+                            ||
+                             (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge")));
           // Delete the data in the partitions which have other locations
           deletePartitionData(partPaths, ifPurge);
           // Delete the data in the table
@@ -2561,15 +2586,31 @@ public class HiveMetaStore extends Thrif
           ms.rollbackTransaction();
         } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) {
           if (tbl != null && !isExternal(tbl)) {
+            // Data needs deletion. Check if trash may be skipped.
+            // Trash may be skipped iff:
+            //  1. deleteData == true, obviously.
+            //  2. tbl is external.
+            //  3. Either
+            //    3.1. User has specified PURGE from the commandline, and if not,
+            //    3.2. User has set the table to auto-purge.
+            boolean mustPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
+                                ||
+                                 (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge")));
+            if (mustPurge) {
+              LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash.");
+            }
+            else {
+              LOG.info("dropPartition() will move " + partPath + " to trash-directory.");
+            }
             // Archived partitions have har:/to_har_file as their location.
             // The original directory was saved in params
             if (isArchived) {
               assert (archiveParentDir != null);
-              wh.deleteDir(archiveParentDir, true);
+              wh.deleteDir(archiveParentDir, true, mustPurge);
             } else {
               assert (partPath != null);
-              wh.deleteDir(partPath, true);
-              deleteParentRecursive(partPath.getParent(), part_vals.size() - 1);
+              wh.deleteDir(partPath, true, mustPurge);
+              deleteParentRecursive(partPath.getParent(), part_vals.size() - 1, mustPurge);
             }
             // ok even if the data is not deleted
           }
@@ -2584,10 +2625,10 @@ public class HiveMetaStore extends Thrif
       return true;
     }
 
-    private void deleteParentRecursive(Path parent, int depth) throws IOException, MetaException {
+    private void deleteParentRecursive(Path parent, int depth, boolean mustPurge) throws IOException, MetaException {
       if (depth > 0 && parent != null && wh.isWritable(parent) && wh.isEmpty(parent)) {
-        wh.deleteDir(parent, true);
-        deleteParentRecursive(parent.getParent(), depth - 1);
+        wh.deleteDir(parent, true, mustPurge);
+        deleteParentRecursive(parent.getParent(), depth - 1, mustPurge);
       }
     }
 
@@ -2714,15 +2755,28 @@ public class HiveMetaStore extends Thrif
         if (!success) {
           ms.rollbackTransaction();
         } else if (deleteData && !isExternal(tbl)) {
+          // Data needs deletion. Check if trash may be skipped.
+          // Trash may be skipped iff:
+          //  1. deleteData == true, obviously.
+          //  2. tbl is external.
+          //  3. Either
+          //    3.1. User has specified PURGE from the commandline, and if not,
+          //    3.2. User has set the table to auto-purge.
+          boolean mustPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
+                              ||
+                              (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge")));
+          LOG.info( mustPurge?
+                      "dropPartition() will purge partition-directories directly, skipping trash."
+                    :  "dropPartition() will move partition-directories to trash-directory.");
           // Archived partitions have har:/to_har_file as their location.
           // The original directory was saved in params
           for (Path path : archToDelete) {
-            wh.deleteDir(path, true);
+            wh.deleteDir(path, true, mustPurge);
           }
           for (PathAndPartValSize p : dirsToDelete) {
-            wh.deleteDir(p.path, true);
+            wh.deleteDir(p.path, true, mustPurge);
             try {
-              deleteParentRecursive(p.path.getParent(), p.partValSize - 1);
+              deleteParentRecursive(p.path.getParent(), p.partValSize - 1, mustPurge);
             } catch (IOException ex) {
               LOG.warn("Error from deleteParentRecursive", ex);
               throw new MetaException("Failed to delete parent: " + ex.getMessage());
@@ -4088,12 +4142,16 @@ public class HiveMetaStore extends Thrif
     @Override
     public TableStatsResult get_table_statistics_req(TableStatsRequest request)
         throws MetaException, NoSuchObjectException, TException {
-      String dbName = request.getDbName(), tblName = request.getTblName();
+      String dbName = request.getDbName().toLowerCase();
+      String tblName = request.getTblName().toLowerCase();
       startFunction("get_table_statistics_req: db=" + dbName + " table=" + tblName);
       TableStatsResult result = null;
+      List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size());
+      for (String colName : request.getColNames()) {
+        lowerCaseColNames.add(colName.toLowerCase());
+      }
       try {
-        ColumnStatistics cs = getMS().getTableColumnStatistics(
-            dbName, tblName, request.getColNames());
+        ColumnStatistics cs = getMS().getTableColumnStatistics(dbName, tblName, lowerCaseColNames);
         result = new TableStatsResult(
             cs == null ? Lists.<ColumnStatisticsObj>newArrayList() : cs.getStatsObj());
       } finally {
@@ -4131,13 +4189,22 @@ public class HiveMetaStore extends Thrif
     @Override
     public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsRequest request)
         throws MetaException, NoSuchObjectException, TException {
-      String dbName = request.getDbName(), tblName = request.getTblName();
+      String dbName = request.getDbName().toLowerCase();
+      String tblName = request.getTblName().toLowerCase();
       startFunction("get_partitions_statistics_req: db=" + dbName + " table=" + tblName);
 
       PartitionsStatsResult result = null;
+      List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size());
+      for (String colName : request.getColNames()) {
+        lowerCaseColNames.add(colName.toLowerCase());
+      }
+      List<String> lowerCasePartNames = new ArrayList<String>(request.getPartNames().size());
+      for (String partName : request.getPartNames()) {
+        lowerCasePartNames.add(lowerCaseConvertPartName(partName));
+      }
       try {
         List<ColumnStatistics> stats = getMS().getPartitionColumnStatistics(
-            dbName, tblName, request.getPartNames(), request.getColNames());
+            dbName, tblName, lowerCasePartNames, lowerCaseColNames);
         Map<String, List<ColumnStatisticsObj>> map =
             new HashMap<String, List<ColumnStatisticsObj>>();
         for (ColumnStatistics stat : stats) {
@@ -5525,11 +5592,23 @@ public class HiveMetaStore extends Thrif
     @Override
     public AggrStats get_aggr_stats_for(PartitionsStatsRequest request)
         throws NoSuchObjectException, MetaException, TException {
+      String dbName = request.getDbName().toLowerCase();
+      String tblName = request.getTblName().toLowerCase();
       startFunction("get_aggr_stats_for: db=" + request.getDbName() + " table=" + request.getTblName());
+
+      List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size());
+      for (String colName : request.getColNames()) {
+        lowerCaseColNames.add(colName.toLowerCase());
+      }
+      List<String> lowerCasePartNames = new ArrayList<String>(request.getPartNames().size());
+      for (String partName : request.getPartNames()) {
+        lowerCasePartNames.add(lowerCaseConvertPartName(partName));
+      }
       AggrStats aggrStats = null;
+
       try {
-        aggrStats = new AggrStats(getMS().get_aggr_stats_for(request.getDbName(),
-          request.getTblName(), request.getPartNames(), request.getColNames()));
+        aggrStats = new AggrStats(getMS().get_aggr_stats_for(dbName, tblName, lowerCasePartNames,
+            lowerCaseColNames));
         return aggrStats;
       } finally {
           endFunction("get_partitions_statistics_req: ", aggrStats == null, null, request.getTblName());

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Tue Mar 10 04:20:45 2015
@@ -34,6 +34,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -729,7 +730,6 @@ public class HiveMetaStoreClient impleme
     client.drop_database(name, deleteData, cascade);
   }
 
-
   /**
    * @param tbl_name
    * @param db_name
@@ -758,6 +758,21 @@ public class HiveMetaStoreClient impleme
     return dropPartition(dbName, tableName, partName, deleteData, null);
   }
 
+  private static EnvironmentContext getEnvironmentContextWithIfPurgeSet() {
+    Map<String, String> warehouseOptions = new HashMap<String, String>();
+    warehouseOptions.put("ifPurge", "TRUE");
+    return new EnvironmentContext(warehouseOptions);
+  }
+
+  /*
+  public boolean dropPartition(String dbName, String tableName, String partName, boolean deleteData, boolean ifPurge)
+      throws NoSuchObjectException, MetaException, TException {
+
+    return dropPartition(dbName, tableName, partName, deleteData,
+                         ifPurge? getEnvironmentContextWithIfPurgeSet() : null);
+  }
+  */
+
   public boolean dropPartition(String dbName, String tableName, String partName, boolean deleteData,
       EnvironmentContext envContext) throws NoSuchObjectException, MetaException, TException {
     return client.drop_partition_by_name_with_environment_context(dbName, tableName, partName,
@@ -784,6 +799,13 @@ public class HiveMetaStoreClient impleme
     return dropPartition(db_name, tbl_name, part_vals, deleteData, null);
   }
 
+  @Override
+  public boolean dropPartition(String db_name, String tbl_name,
+      List<String> part_vals, PartitionDropOptions options) throws TException {
+    return dropPartition(db_name, tbl_name, part_vals, options.deleteData,
+                         options.purgeData? getEnvironmentContextWithIfPurgeSet() : null);
+  }
+
   public boolean dropPartition(String db_name, String tbl_name, List<String> part_vals,
       boolean deleteData, EnvironmentContext envContext) throws NoSuchObjectException,
       MetaException, TException {
@@ -793,8 +815,8 @@ public class HiveMetaStoreClient impleme
 
   @Override
   public List<Partition> dropPartitions(String dbName, String tblName,
-      List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, boolean ignoreProtection,
-      boolean ifExists, boolean needResult) throws NoSuchObjectException, MetaException, TException {
+                                        List<ObjectPair<Integer, byte[]>> partExprs, PartitionDropOptions options)
+      throws TException {
     RequestPartsSpec rps = new RequestPartsSpec();
     List<DropPartitionsExpr> exprs = new ArrayList<DropPartitionsExpr>(partExprs.size());
     for (ObjectPair<Integer, byte[]> partExpr : partExprs) {
@@ -805,19 +827,41 @@ public class HiveMetaStoreClient impleme
     }
     rps.setExprs(exprs);
     DropPartitionsRequest req = new DropPartitionsRequest(dbName, tblName, rps);
-    req.setDeleteData(deleteData);
-    req.setIgnoreProtection(ignoreProtection);
-    req.setNeedResult(needResult);
-    req.setIfExists(ifExists);
+    req.setDeleteData(options.deleteData);
+    req.setIgnoreProtection(options.ignoreProtection);
+    req.setNeedResult(options.returnResults);
+    req.setIfExists(options.ifExists);
+    if (options.purgeData) {
+      LOG.info("Dropped partitions will be purged!");
+      req.setEnvironmentContext(getEnvironmentContextWithIfPurgeSet());
+    }
     return client.drop_partitions_req(req).getPartitions();
   }
 
   @Override
   public List<Partition> dropPartitions(String dbName, String tblName,
       List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, boolean ignoreProtection,
+      boolean ifExists, boolean needResult) throws NoSuchObjectException, MetaException, TException {
+
+    return dropPartitions(dbName, tblName, partExprs,
+                          PartitionDropOptions.instance()
+                                              .deleteData(deleteData)
+                                              .ignoreProtection(ignoreProtection)
+                                              .ifExists(ifExists)
+                                              .returnResults(needResult));
+
+  }
+
+  @Override
+  public List<Partition> dropPartitions(String dbName, String tblName,
+      List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, boolean ignoreProtection,
       boolean ifExists) throws NoSuchObjectException, MetaException, TException {
     // By default, we need the results from dropPartitions();
-    return dropPartitions(dbName, tblName, partExprs, deleteData, ignoreProtection, ifExists, true);
+    return dropPartitions(dbName, tblName, partExprs,
+                          PartitionDropOptions.instance()
+                                              .deleteData(deleteData)
+                                              .ignoreProtection(ignoreProtection)
+                                              .ifExists(ifExists));
   }
 
   /**

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Tue Mar 10 04:20:45 2015
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.common.Val
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.CompactionType;
 import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
 import org.apache.hadoop.hive.metastore.api.FireEventRequest;
 import org.apache.hadoop.hive.metastore.api.FireEventResponse;
 import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
@@ -660,6 +661,19 @@ public interface IMetaStoreClient {
       List<String> part_vals, boolean deleteData) throws NoSuchObjectException,
       MetaException, TException;
 
+  /**
+   * Method to dropPartitions() with the option to purge the partition data directly,
+   * rather than to move data to trash.
+   * @param db_name Name of the database.
+   * @param tbl_name Name of the table.
+   * @param part_vals Specification of the partitions being dropped.
+   * @param options PartitionDropOptions for the operation.
+   * @return True (if partitions are dropped), else false.
+   * @throws TException
+   */
+  boolean dropPartition(String db_name, String tbl_name, List<String> part_vals,
+                        PartitionDropOptions options) throws TException;
+
   List<Partition> dropPartitions(String dbName, String tblName,
       List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, boolean ignoreProtection,
       boolean ifExists) throws NoSuchObjectException, MetaException, TException;
@@ -668,6 +682,18 @@ public interface IMetaStoreClient {
       List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, boolean ignoreProtection,
       boolean ifExists, boolean needResults) throws NoSuchObjectException, MetaException, TException;
 
+  /**
+   * Generalization of dropPartitions(),
+   * @param dbName Name of the database
+   * @param tblName Name of the table
+   * @param partExprs Partition-specification
+   * @param options Boolean options for dropping partitions
+   * @return List of Partitions dropped
+   * @throws TException On failure
+   */
+  List<Partition> dropPartitions(String dbName, String tblName,
+                                 List<ObjectPair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws TException;
+
   boolean dropPartition(String db_name, String tbl_name,
       String name, boolean deleteData) throws NoSuchObjectException,
       MetaException, TException;

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Tue Mar 10 04:20:45 2015
@@ -32,7 +32,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
-import javax.jdo.JDODataStoreException;
 import javax.jdo.PersistenceManager;
 import javax.jdo.Query;
 import javax.jdo.Transaction;
@@ -41,7 +40,6 @@ import javax.jdo.datastore.JDOConnection
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.derby.iapi.error.StandardException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -497,6 +495,7 @@ class MetaStoreDirectSql {
     @SuppressWarnings("unchecked")
     List<Object[]> sqlResult = executeWithArray(query, null, queryText);
     long queryTime = doTrace ? System.nanoTime() : 0;
+    Deadline.checkTimeout();
 
     // Read all the fields and create partitions, SDs and serdes.
     TreeMap<Long, Partition> partitions = new TreeMap<Long, Partition>();
@@ -585,6 +584,7 @@ class MetaStoreDirectSql {
       serde.setSerializationLib((String)fields[13]);
       serdeSb.append(serdeId).append(",");
       sd.setSerdeInfo(serde);
+      Deadline.checkTimeout();
     }
     query.closeAll();
     timingTrace(doTrace, queryText, start, queryTime);

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java Tue Mar 10 04:20:45 2015
@@ -86,6 +86,10 @@ public class RetryingHMSHandler implemen
         HiveConf.ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS);
     int retryLimit = HiveConf.getIntVar(origConf,
         HiveConf.ConfVars.HMSHANDLERATTEMPTS);
+    long timeout = HiveConf.getTimeVar(origConf,
+        HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);
+
+    Deadline.registerIfNot(timeout);
 
     if (reloadConf) {
       MetaStoreInit.updateConnectionURL(origConf, getActiveConf(),
@@ -99,7 +103,10 @@ public class RetryingHMSHandler implemen
         if (reloadConf || gotNewConnectUrl) {
           baseHandler.setConf(getActiveConf());
         }
-        return method.invoke(baseHandler, args);
+        Deadline.startTimer(method.getName());
+        Object object = method.invoke(baseHandler, args);
+        Deadline.stopTimer();
+        return object;
 
       } catch (javax.jdo.JDOException e) {
         caughtException = e;
@@ -132,11 +139,21 @@ public class RetryingHMSHandler implemen
             LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
           }
           throw e.getCause();
-        } else if (e.getCause() instanceof MetaException && e.getCause().getCause() != null
-            && (e.getCause().getCause() instanceof javax.jdo.JDOException || 
-            	e.getCause().getCause() instanceof NucleusException)) {
-          // The JDOException or the Nucleus Exception may be wrapped further in a MetaException
-          caughtException = e.getCause().getCause();
+        } else if (e.getCause() instanceof MetaException && e.getCause().getCause() != null) {
+          if (e.getCause().getCause() instanceof javax.jdo.JDOException ||
+              e.getCause().getCause() instanceof NucleusException) {
+            // The JDOException or the Nucleus Exception may be wrapped further in a MetaException
+            caughtException = e.getCause().getCause();
+          } else if (e.getCause().getCause() instanceof DeadlineException) {
+            // The Deadline Exception needs no retry and be thrown immediately.
+            Deadline.clear();
+            LOG.error("Error happens in method " + method.getName() + ": " +
+                ExceptionUtils.getStackTrace(e.getCause()));
+            throw e.getCause();
+          } else {
+            LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
+            throw e.getCause();
+          }
         } else {
           LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
           throw e.getCause();

Modified: hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/branches/spark/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Tue Mar 10 04:20:45 2015
@@ -32,9 +32,7 @@ import java.util.Map.Entry;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import javax.security.auth.login.LoginException;
 
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -55,8 +53,6 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 
 /**
@@ -250,41 +246,25 @@ public class Warehouse {
       return false;
     }
     final FileStatus stat;
+    final FileSystem fs;
     try {
-      stat = getFs(path).getFileStatus(path);
+      fs = getFs(path);
+      stat = fs.getFileStatus(path);
+      ShimLoader.getHadoopShims().checkFileAccess(fs, stat, FsAction.WRITE);
+      return true;
     } catch (FileNotFoundException fnfe){
       // File named by path doesn't exist; nothing to validate.
       return true;
     } catch (Exception e) {
       // all other exceptions are considered as emanating from
       // unauthorized accesses
-      return false;
-    }
-    final UserGroupInformation ugi;
-    try {
-      ugi = Utils.getUGI();
-    } catch (LoginException le) {
-      throw new IOException(le);
-    }
-    String user = ugi.getShortUserName();
-    //check whether owner can delete
-    if (stat.getOwner().equals(user) &&
-        stat.getPermission().getUserAction().implies(FsAction.WRITE)) {
-      return true;
-    }
-    //check whether group of the user can delete
-    if (stat.getPermission().getGroupAction().implies(FsAction.WRITE)) {
-      String[] groups = ugi.getGroupNames();
-      if (ArrayUtils.contains(groups, stat.getGroup())) {
-        return true;
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Exception when checking if path (" + path + ")", e);
       }
+      return false;
     }
-    //check whether others can delete (uncommon case!!)
-    if (stat.getPermission().getOtherAction().implies(FsAction.WRITE)) {
-      return true;
-    }
-    return false;
   }
+
   /*
   // NOTE: This is for generating the internal path name for partitions. Users
   // should always use the MetaStore API to get the path name for a partition.

Modified: hive/branches/spark/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/pom.xml?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/pom.xml (original)
+++ hive/branches/spark/pom.xml Tue Mar 10 04:20:45 2015
@@ -121,8 +121,8 @@
     <hadoop-20S.version>1.2.1</hadoop-20S.version>
     <hadoop-23.version>2.6.0</hadoop-23.version>
     <hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>
-    <hbase.hadoop1.version>0.98.3-hadoop1</hbase.hadoop1.version>
-    <hbase.hadoop2.version>0.98.3-hadoop2</hbase.hadoop2.version>
+    <hbase.hadoop1.version>0.98.9-hadoop1</hbase.hadoop1.version>
+    <hbase.hadoop2.version>0.98.9-hadoop2</hbase.hadoop2.version>
     <!-- httpcomponents are not always in version sync -->
     <httpcomponents.client.version>4.2.5</httpcomponents.client.version>
     <httpcomponents.core.version>4.2.5</httpcomponents.core.version>

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java Tue Mar 10 04:20:45 2015
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql;
 
+
 /**
  *
  * QueryProperties.
@@ -25,16 +26,26 @@ package org.apache.hadoop.hive.ql;
  * A structure to contain features of a query that are determined
  * during parsing and may be useful for categorizing a query type
  *
- * These inlucde whether the query contains:
+ * These include whether the query contains:
  * a join clause, a group by clause, an order by clause, a sort by
  * clause, a group by clause following a join clause, and whether
  * the query uses a script for mapping/reducing
  */
 public class QueryProperties {
 
+  boolean query;
+  boolean analyzeCommand;
+  boolean partialScanAnalyzeCommand;
+  boolean noScanAnalyzeCommand;
+  boolean analyzeRewrite;
+  boolean ctas;
+  boolean insertToTable;
+  int outerQueryLimit;
+
   boolean hasJoin = false;
   boolean hasGroupBy = false;
   boolean hasOrderBy = false;
+  boolean hasOuterOrderBy = false;
   boolean hasSortBy = false;
   boolean hasJoinFollowedByGroupBy = false;
   boolean hasPTF = false;
@@ -51,10 +62,75 @@ public class QueryProperties {
   private int noOfJoins = 0;
   private int noOfOuterJoins = 0;
   private boolean hasLateralViews;
-  
+
   private boolean multiDestQuery;
   private boolean filterWithSubQuery;
-  
+
+
+  public boolean isQuery() {
+    return query;
+  }
+
+  public void setQuery(boolean query) {
+    this.query = query;
+  }
+
+  public boolean isAnalyzeCommand() {
+    return analyzeCommand;
+  }
+
+  public void setAnalyzeCommand(boolean analyzeCommand) {
+    this.analyzeCommand = analyzeCommand;
+  }
+
+  public boolean isPartialScanAnalyzeCommand() {
+    return partialScanAnalyzeCommand;
+  }
+
+  public void setPartialScanAnalyzeCommand(boolean partialScanAnalyzeCommand) {
+    this.partialScanAnalyzeCommand = partialScanAnalyzeCommand;
+  }
+
+  public boolean isNoScanAnalyzeCommand() {
+    return noScanAnalyzeCommand;
+  }
+
+  public void setNoScanAnalyzeCommand(boolean noScanAnalyzeCommand) {
+    this.noScanAnalyzeCommand = noScanAnalyzeCommand;
+  }
+
+  public boolean isAnalyzeRewrite() {
+    return analyzeRewrite;
+  }
+
+  public void setAnalyzeRewrite(boolean analyzeRewrite) {
+    this.analyzeRewrite = analyzeRewrite;
+  }
+
+  public boolean isCTAS() {
+    return ctas;
+  }
+
+  public void setCTAS(boolean ctas) {
+    this.ctas = ctas;
+  }
+
+  public boolean isInsertToTable() {
+    return insertToTable;
+  }
+
+  public void setInsertToTable(boolean insertToTable) {
+    this.insertToTable = insertToTable;
+  }
+
+  public int getOuterQueryLimit() {
+    return outerQueryLimit;
+  }
+
+  public void setOuterQueryLimit(int outerQueryLimit) {
+    this.outerQueryLimit = outerQueryLimit;
+  }
+
   public boolean hasJoin() {
     return (noOfJoins > 0);
   }
@@ -97,6 +173,14 @@ public class QueryProperties {
     this.hasOrderBy = hasOrderBy;
   }
 
+  public boolean hasOuterOrderBy() {
+    return hasOuterOrderBy;
+  }
+
+  public void setHasOuterOrderBy(boolean hasOuterOrderBy) {
+    this.hasOuterOrderBy = hasOuterOrderBy;
+  }
+
   public boolean hasSortBy() {
     return hasSortBy;
   }
@@ -186,9 +270,19 @@ public class QueryProperties {
   }
 
   public void clear() {
+    query = false;
+    analyzeCommand = false;
+    partialScanAnalyzeCommand = false;
+    noScanAnalyzeCommand = false;
+    analyzeRewrite = false;
+    ctas = false;
+    insertToTable = false;
+    outerQueryLimit = -1;
+
     hasJoin = false;
     hasGroupBy = false;
     hasOrderBy = false;
+    hasOuterOrderBy = false;
     hasSortBy = false;
     hasJoinFollowedByGroupBy = false;
     hasPTF = false;
@@ -204,8 +298,9 @@ public class QueryProperties {
 
     noOfJoins = 0;
     noOfOuterJoins = 0;
-    
+
     multiDestQuery = false;
     filterWithSubQuery = false;
   }
+
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java Tue Mar 10 04:20:45 2015
@@ -48,9 +48,6 @@ public abstract class AbstractMapJoinOpe
 
   transient int numMapRowsRead;
 
-  transient boolean firstRow;
-
-
   public AbstractMapJoinOperator() {
   }
 
@@ -72,7 +69,6 @@ public abstract class AbstractMapJoinOpe
     super.initializeOp(hconf);
 
     numMapRowsRead = 0;
-    firstRow = true;
 
     // all other tables are small, and are cached in the hash table
     posBigTable = (byte) conf.getPosBigTable();

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java Tue Mar 10 04:20:45 2015
@@ -500,8 +500,11 @@ public class CommonMergeJoinOperator ext
     }
     Map<Integer, DummyStoreOperator> dummyOps = parent.getTagToOperatorTree();
     for (Entry<Integer, DummyStoreOperator> connectOp : dummyOps.entrySet()) {
-      parentOperators.add(connectOp.getKey(), connectOp.getValue());
-      connectOp.getValue().getChildOperators().add(this);
+      if (connectOp.getValue().getChildOperators() == null
+	  || connectOp.getValue().getChildOperators().isEmpty()) {
+	parentOperators.add(connectOp.getKey(), connectOp.getValue());
+	connectOp.getValue().getChildOperators().add(this);
+      }
     }
     super.initializeLocalWork(hconf);
     return;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Tue Mar 10 04:20:45 2015
@@ -61,6 +61,7 @@ import org.apache.hadoop.hive.common.typ
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.PartitionDropOptions;
 import org.apache.hadoop.hive.metastore.ProtectMode;
 import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.Warehouse;
@@ -3679,8 +3680,14 @@ public class DDLTask extends Task<DDLWor
 
   private void dropPartitions(Hive db, Table tbl, DropTableDesc dropTbl) throws HiveException {
     // ifExists is currently verified in DDLSemanticAnalyzer
-    List<Partition> droppedParts = db.dropPartitions(dropTbl.getTableName(),
-        dropTbl.getPartSpecs(), true, dropTbl.getIgnoreProtection(), true);
+    List<Partition> droppedParts
+        = db.dropPartitions(dropTbl.getTableName(),
+                            dropTbl.getPartSpecs(),
+                            PartitionDropOptions.instance()
+                                                .deleteData(true)
+                                                .ignoreProtection(dropTbl.getIgnoreProtection())
+                                                .ifExists(true)
+                                                .purgeData(dropTbl.getIfPurge()));
     for (Partition partition : droppedParts) {
       console.printInfo("Dropped the partition " + partition.getName());
       // We have already locked the table, don't lock the partitions.

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue Mar 10 04:20:45 2015
@@ -180,7 +180,8 @@ public final class FunctionRegistry {
     system.registerUDF("ascii", UDFAscii.class, false);
     system.registerGenericUDF("lpad", GenericUDFLpad.class);
     system.registerGenericUDF("rpad", GenericUDFRpad.class);
-    system.registerGenericUDF("levenshtein", GenericUDFLevenstein.class);
+    system.registerGenericUDF("levenshtein", GenericUDFLevenshtein.class);
+    system.registerGenericUDF("soundex", GenericUDFSoundex.class);
 
     system.registerGenericUDF("size", GenericUDFSize.class);
 
@@ -261,6 +262,7 @@ public final class FunctionRegistry {
     system.registerUDF("weekofyear", UDFWeekOfYear.class, false);
     system.registerGenericUDF("last_day", GenericUDFLastDay.class);
     system.registerGenericUDF("next_day", GenericUDFNextDay.class);
+    system.registerGenericUDF("trunc", GenericUDFTrunc.class);
 
     system.registerGenericUDF("date_add", GenericUDFDateAdd.class);
     system.registerGenericUDF("date_sub", GenericUDFDateSub.class);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Tue Mar 10 04:20:45 2015
@@ -21,10 +21,14 @@ package org.apache.hadoop.hive.ql.exec;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Callable;
 
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.HashTableLoaderFactory;
 import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionHandler;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
@@ -55,8 +59,7 @@ public class MapJoinOperator extends Abs
   private static final String CLASS_NAME = MapJoinOperator.class.getName();
   private final PerfLogger perfLogger = PerfLogger.getPerfLogger();
 
-  private transient String tableKey;
-  private transient String serdeKey;
+  private transient String cacheKey;
   private transient ObjectCache cache;
 
   protected HashTableLoader loader;
@@ -99,28 +102,53 @@ public class MapJoinOperator extends Abs
 
     // On Tez only: The hash map might already be cached in the container we run
     // the task in. On MR: The cache is a no-op.
-    tableKey = "__HASH_MAP_"+this.getOperatorId()+"_container";
-    serdeKey = "__HASH_MAP_"+this.getOperatorId()+"_serde";
+    cacheKey = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVEQUERYID)
+      + "__HASH_MAP_"+this.getOperatorId()+"_container";
 
     cache = ObjectCacheFactory.getCache(hconf);
     loader = HashTableLoaderFactory.getLoader(hconf);
 
     hashMapRowGetters = null;
 
-    mapJoinTables = (MapJoinTableContainer[]) cache.retrieve(tableKey);
-    mapJoinTableSerdes = (MapJoinTableContainerSerDe[]) cache.retrieve(serdeKey);
-    hashTblInitedOnce = true;
-    if (isLogInfoEnabled) {
-      LOG.info("Try to retrieve from cache");
-    }
+    mapJoinTables = new MapJoinTableContainer[tagLen];
+    mapJoinTableSerdes = new MapJoinTableContainerSerDe[tagLen];
+    hashTblInitedOnce = false;
 
-    if (mapJoinTables == null || mapJoinTableSerdes == null) {
+    generateMapMetaData();
+
+    if (!conf.isBucketMapJoin()) {
+      /*
+       * The issue with caching in case of bucket map join is that different tasks
+       * process different buckets and if the container is reused to join a different bucket,
+       * join results can be incorrect. The cache is keyed on operator id and for bucket map join
+       * the operator does not change but data needed is different. For a proper fix, this
+       * requires changes in the Tez API with regard to finding bucket id and
+       * also ability to schedule tasks to re-use containers that have cached the specific bucket.
+       */
       if (isLogInfoEnabled) {
-	LOG.info("Did not find tables in cache");
+	LOG.info("This is not bucket map join, so cache");
       }
-      mapJoinTables = new MapJoinTableContainer[tagLen];
-      mapJoinTableSerdes = new MapJoinTableContainerSerDe[tagLen];
-      hashTblInitedOnce = false;
+
+      Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> pair =
+	(Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>)
+	cache.retrieve(cacheKey, new Callable<Object>() {
+	  public Object call() throws HiveException {
+	    return loadHashTable();
+	  }
+	});
+
+      mapJoinTables = pair.getLeft();
+      mapJoinTableSerdes = pair.getRight();
+      hashTblInitedOnce = true;
+    } else {
+      loadHashTable();
+    }
+
+    if (this.getExecContext() != null) {
+      // reset exec context so that initialization of the map operator happens
+      // poperly
+      this.getExecContext().setLastInputPath(null);
+      this.getExecContext().setCurrentInputPath(null);
     }
   }
 
@@ -147,85 +175,71 @@ public class MapJoinOperator extends Abs
     return valueOI;
   }
 
-  public void generateMapMetaData() throws HiveException, SerDeException {
+  public void generateMapMetaData() throws HiveException {
     // generate the meta data for key
     // index for key is -1
 
-    TableDesc keyTableDesc = conf.getKeyTblDesc();
-    SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
-        null);
-    SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null);
-    MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false);
-    for (int pos = 0; pos < order.length; pos++) {
-      if (pos == posBigTable) {
-        continue;
-      }
-      TableDesc valueTableDesc;
-      if (conf.getNoOuterJoin()) {
-        valueTableDesc = conf.getValueTblDescs().get(pos);
-      } else {
-        valueTableDesc = conf.getValueFilteredTblDescs().get(pos);
+    try {
+      TableDesc keyTableDesc = conf.getKeyTblDesc();
+      SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
+	  null);
+      SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null);
+      MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false);
+      for (int pos = 0; pos < order.length; pos++) {
+	if (pos == posBigTable) {
+	  continue;
+	}
+	TableDesc valueTableDesc;
+	if (conf.getNoOuterJoin()) {
+	  valueTableDesc = conf.getValueTblDescs().get(pos);
+	} else {
+	  valueTableDesc = conf.getValueFilteredTblDescs().get(pos);
+	}
+	SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(),
+	    null);
+	SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);
+	MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos));
+	mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext);
       }
-      SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(),
-          null);
-      SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);
-      MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos));
-      mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext);
+    } catch (SerDeException e) {
+      throw new HiveException(e);
     }
   }
 
-  private void loadHashTable() throws HiveException {
+  private Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>
+    loadHashTable() throws HiveException {
 
-    if ((this.getExecContext() == null)
-        || (this.getExecContext().getLocalWork() == null)
-        || (this.getExecContext().getLocalWork().getInputFileChangeSensitive() == false)
-    ) {
-      /*
-       * This early-exit criteria is not applicable if the local work is sensitive to input file changes.
-       * But the check does no apply if there is no local work, or if this is a reducer vertex (execContext is null).
-       */
-      if (hashTblInitedOnce) {
-        return;
-      } else {
-        hashTblInitedOnce = true;
-      }
+    if (this.hashTblInitedOnce
+	&& ((this.getExecContext() == null)
+	    || (this.getExecContext().getLocalWork() == null)
+	    || (this.getExecContext().getLocalWork().getInputFileChangeSensitive()
+		== false))) {
+      // no need to reload
+      return new ImmutablePair<MapJoinTableContainer[],
+	MapJoinTableContainerSerDe[]> (mapJoinTables, mapJoinTableSerdes);
     }
+
     perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.LOAD_HASHTABLE);
     loader.init(getExecContext(), hconf, this);
     long memUsage = (long)(MapJoinMemoryExhaustionHandler.getMaxHeapSize()
         * conf.getHashTableMemoryUsage());
     loader.load(mapJoinTables, mapJoinTableSerdes, memUsage);
-    if (!conf.isBucketMapJoin()) {
-      /*
-       * The issue with caching in case of bucket map join is that different tasks
-       * process different buckets and if the container is reused to join a different bucket,
-       * join results can be incorrect. The cache is keyed on operator id and for bucket map join
-       * the operator does not change but data needed is different. For a proper fix, this
-       * requires changes in the Tez API with regard to finding bucket id and
-       * also ability to schedule tasks to re-use containers that have cached the specific bucket.
-       */
-      if (isLogInfoEnabled) {
-	LOG.info("This is not bucket map join, so cache");
-      }
-      cache.cache(tableKey, mapJoinTables);
-      cache.cache(serdeKey, mapJoinTableSerdes);
-    }
+
+    hashTblInitedOnce = true;
+
+    Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> pair
+      = new ImmutablePair<MapJoinTableContainer[],
+      MapJoinTableContainerSerDe[]> (mapJoinTables, mapJoinTableSerdes);
+
     perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.LOAD_HASHTABLE);
+
+    return pair;
   }
 
   // Load the hash table
   @Override
   public void cleanUpInputFileChangedOp() throws HiveException {
-    try {
-      if (firstRow) {
-        // generate the map metadata
-        generateMapMetaData();
-        firstRow = false;
-      }
-      loadHashTable();
-    } catch (SerDeException e) {
-      throw new HiveException(e);
-    }
+    loadHashTable();
   }
 
   protected void setMapJoinKey(
@@ -248,12 +262,6 @@ public class MapJoinOperator extends Abs
   @Override
   public void processOp(Object row, int tag) throws HiveException {
     try {
-      if (firstRow) {
-        generateMapMetaData();
-        loadHashTable();
-        firstRow = false;
-      }
-
       alias = (byte) tag;
       if (hashMapRowGetters == null) {
         hashMapRowGetters = new ReusableGetAdaptor[mapJoinTables.length];
@@ -337,6 +345,7 @@ public class MapJoinOperator extends Abs
         }
       }
     }
+    cache.release(cacheKey);
     super.closeOp(abort);
   }
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCache.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCache.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCache.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCache.java Tue Mar 10 04:20:45 2015
@@ -17,21 +17,24 @@
  */
 package org.apache.hadoop.hive.ql.exec;
 
+import java.util.concurrent.Callable;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
 /**
  * ObjectCache. Interface for maintaining objects associated with a task.
  */
 public interface ObjectCache {
+
   /**
-   * Add an object to the cache
    * @param key
-   * @param value
    */
-  public void cache(String key, Object value);
+  public void release(String key);
 
   /**
    * Retrieve object from cache.
    * @param key
+   * @param fn function to generate the object if it's not there
    * @return the last cached object with the key, null if none.
    */
-  public Object retrieve(String key);
+  public Object retrieve(String key, Callable<?> fn) throws HiveException;
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Tue Mar 10 04:20:45 2015
@@ -101,8 +101,6 @@ public class SMBMapJoinOperator extends
 
     super.initializeOp(hconf);
 
-    firstRow = true;
-
     closeCalled = false;
 
     this.firstFetchHappened = false;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java Tue Mar 10 04:20:45 2015
@@ -38,6 +38,7 @@ import org.apache.hadoop.security.token.
  */
 public class SecureCmdDoAs {
   private final Path tokenPath;
+  private final File tokenFile;
 
   public SecureCmdDoAs(HiveConf conf) throws HiveException, IOException{
     // Get delegation token for user from filesystem and write the token along with
@@ -46,8 +47,8 @@ public class SecureCmdDoAs {
     FileSystem fs = FileSystem.get(conf);
     Token<?> fsToken = fs.getDelegationToken(uname);
 
-    File t = File.createTempFile("hive_hadoop_delegation_token", null);
-    tokenPath = new Path(t.toURI());
+    tokenFile = File.createTempFile("hive_hadoop_delegation_token", null);
+    tokenPath = new Path(tokenFile.toURI());
 
     //write credential with token to file
     Credentials cred = new Credentials();
@@ -60,4 +61,7 @@ public class SecureCmdDoAs {
         tokenPath.toUri().getPath());
   }
 
+  public void close() {
+    tokenFile.delete();
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java Tue Mar 10 04:20:45 2015
@@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.io.Stat
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
 import org.apache.hadoop.hive.ql.plan.StatsNoJobWork;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
 import org.apache.hadoop.hive.shims.ShimLoader;
@@ -377,7 +377,7 @@ public class StatsNoJobTask extends Task
 
   private List<Partition> getPartitionsList() throws HiveException {
     if (work.getTableSpecs() != null) {
-      tableSpec tblSpec = work.getTableSpecs();
+      TableSpec tblSpec = work.getTableSpecs();
       table = tblSpec.tableHandle;
       if (!table.isPartitioned()) {
         return null;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Tue Mar 10 04:20:45 2015
@@ -39,7 +39,7 @@ import org.apache.hadoop.hive.ql.ErrorMs
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
 import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
 import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
 import org.apache.hadoop.hive.ql.plan.StatsWork;
@@ -370,7 +370,7 @@ public class StatsTask extends Task<Stat
     if (work.getTableSpecs() != null) {
 
       // ANALYZE command
-      tableSpec tblSpec = work.getTableSpecs();
+      TableSpec tblSpec = work.getTableSpecs();
       table = tblSpec.tableHandle;
       if (!table.isPartitioned()) {
         return null;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Mar 10 04:20:45 2015
@@ -2921,7 +2921,7 @@ public final class Utilities {
 
   public static String now() {
     Calendar cal = Calendar.getInstance();
-    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
+    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     return sdf.format(cal.getTime());
   }
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java Tue Mar 10 04:20:45 2015
@@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.CompressionUtils;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.common.LogUtils;
 import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -239,8 +240,8 @@ public class ExecDriver extends Task<Map
     job.setMapOutputValueClass(BytesWritable.class);
 
     try {
-      job.setPartitionerClass((Class<? extends Partitioner>) (Class.forName(HiveConf.getVar(job,
-          HiveConf.ConfVars.HIVEPARTITIONER))));
+      String partitioner = HiveConf.getVar(job, ConfVars.HIVEPARTITIONER);
+      job.setPartitionerClass((Class<? extends Partitioner>) JavaUtils.loadClass(partitioner));
     } catch (ClassNotFoundException e) {
       throw new RuntimeException(e.getMessage(), e);
     }
@@ -286,7 +287,7 @@ public class ExecDriver extends Task<Map
     LOG.info("Using " + inpFormat);
 
     try {
-      job.setInputFormat((Class<? extends InputFormat>) (Class.forName(inpFormat)));
+      job.setInputFormat((Class<? extends InputFormat>) JavaUtils.loadClass(inpFormat));
     } catch (ClassNotFoundException e) {
       throw new RuntimeException(e.getMessage(), e);
     }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java Tue Mar 10 04:20:45 2015
@@ -30,8 +30,6 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.MapOperator;
 import org.apache.hadoop.hive.ql.exec.MapredContext;
-import org.apache.hadoop.hive.ql.exec.ObjectCache;
-import org.apache.hadoop.hive.ql.exec.ObjectCacheFactory;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorUtils;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -93,19 +91,13 @@ public class ExecMapper extends MapReduc
 
     setDone(false);
 
-    ObjectCache cache = ObjectCacheFactory.getCache(job);
-
     try {
       jc = job;
       execContext.setJc(jc);
+
       // create map and fetch operators
-      MapWork mrwork = (MapWork) cache.retrieve(PLAN_KEY);
-      if (mrwork == null) {
-        mrwork = Utilities.getMapWork(job);
-        cache.cache(PLAN_KEY, mrwork);
-      } else {
-        Utilities.setMapWork(job, mrwork);
-      }
+      MapWork mrwork = Utilities.getMapWork(job);
+
       if (mrwork.getVectorMode()) {
         mo = new VectorMapOperator();
       } else {

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java Tue Mar 10 04:20:45 2015
@@ -30,8 +30,6 @@ import java.util.List;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.MapredContext;
-import org.apache.hadoop.hive.ql.exec.ObjectCache;
-import org.apache.hadoop.hive.ql.exec.ObjectCacheFactory;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.mr.ExecMapper.ReportStats;
@@ -111,14 +109,7 @@ public class ExecReducer extends MapRedu
     }
     jc = job;
 
-    ObjectCache cache = ObjectCacheFactory.getCache(jc);
-    ReduceWork gWork = (ReduceWork) cache.retrieve(PLAN_KEY);
-    if (gWork == null) {
-      gWork = Utilities.getReduceWork(job);
-      cache.cache(PLAN_KEY, gWork);
-    } else {
-      Utilities.setReduceWork(job, gWork);
-    }
+    ReduceWork gWork = Utilities.getReduceWork(job);
 
     reducer = gWork.getReducer();
     reducer.setParentOperators(null); // clear out any parents as reducer is the

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java Tue Mar 10 04:20:45 2015
@@ -64,8 +64,6 @@ import org.apache.hadoop.hive.ql.session
 import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.shims.Utils;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -97,6 +95,7 @@ public class MapredLocalTask extends Tas
   private ExecMapperContext execContext = null;
 
   private Process executor;
+  private SecureCmdDoAs secureDoAs;
 
   public MapredLocalTask() {
     super();
@@ -123,7 +122,7 @@ public class MapredLocalTask extends Tas
 
   public static String now() {
     Calendar cal = Calendar.getInstance();
-    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd hh:mm:ss");
+    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     return sdf.format(cal.getTime());
   }
 
@@ -271,7 +270,7 @@ public class MapredLocalTask extends Tas
         //If kerberos security is enabled, and HS2 doAs is enabled,
         // then additional params need to be set so that the command is run as
         // intended user
-        SecureCmdDoAs secureDoAs = new SecureCmdDoAs(conf);
+        secureDoAs = new SecureCmdDoAs(conf);
         secureDoAs.addEnv(variables);
       }
 
@@ -314,9 +313,12 @@ public class MapredLocalTask extends Tas
 
       return exitVal;
     } catch (Exception e) {
-      e.printStackTrace();
-      LOG.error("Exception: " + e.getMessage());
+      LOG.error("Exception: " + e, e);
       return (1);
+    } finally {
+      if (secureDoAs != null) {
+        secureDoAs.close();
+      }
     }
   }
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ObjectCache.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ObjectCache.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ObjectCache.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ObjectCache.java Tue Mar 10 04:20:45 2015
@@ -18,9 +18,11 @@
 
 package org.apache.hadoop.hive.ql.exec.mr;
 
+import java.util.concurrent.Callable;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
+import org.apache.hadoop.hive.ql.metadata.HiveException;
 
 /**
  * ObjectCache. No-op implementation on MR we don't have a means to reuse
@@ -32,14 +34,16 @@ public class ObjectCache implements org.
   private static final Log LOG = LogFactory.getLog(ObjectCache.class.getName());
 
   @Override
-  public void cache(String key, Object value) {
-    LOG.info("Ignoring cache key: "+key);
+  public void release(String key) {
+    // nothing to do
   }
 
   @Override
-  public Object retrieve(String key) {
-    LOG.info("Ignoring retrieval request: "+key);
-    return null;
+  public Object retrieve(String key, Callable<?> fn) throws HiveException {
+    try {
+      return fn.call();
+    } catch (Exception e) {
+      throw new HiveException(e);
+    }
   }
-
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java Tue Mar 10 04:20:45 2015
@@ -29,6 +29,7 @@ import java.util.Map;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
@@ -194,7 +195,7 @@ public class MapJoinTableContainerSerDe
     try {
       @SuppressWarnings("unchecked")
       Class<? extends MapJoinPersistableTableContainer> clazz =
-          (Class<? extends MapJoinPersistableTableContainer>)Class.forName(name);
+          (Class<? extends MapJoinPersistableTableContainer>) JavaUtils.loadClass(name);
       Constructor<? extends MapJoinPersistableTableContainer> constructor =
           clazz.getDeclaredConstructor(Map.class);
       return constructor.newInstance(metaData);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java Tue Mar 10 04:20:45 2015
@@ -22,8 +22,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.MapOperator;
 import org.apache.hadoop.hive.ql.exec.MapredContext;
-import org.apache.hadoop.hive.ql.exec.ObjectCache;
-import org.apache.hadoop.hive.ql.exec.ObjectCacheFactory;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorUtils;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -68,19 +66,13 @@ public class SparkMapRecordHandler exten
     super.init(job, output, reporter);
 
     isLogInfoEnabled = LOG.isInfoEnabled();
-    ObjectCache cache = ObjectCacheFactory.getCache(job);
 
     try {
       jc = job;
       execContext = new ExecMapperContext(jc);
       // create map and fetch operators
-      MapWork mrwork = (MapWork) cache.retrieve(PLAN_KEY);
-      if (mrwork == null) {
-        mrwork = Utilities.getMapWork(job);
-        cache.cache(PLAN_KEY, mrwork);
-      } else {
-        Utilities.setMapWork(job, mrwork);
-      }
+      MapWork mrwork = Utilities.getMapWork(job);
+
       if (mrwork.getVectorMode()) {
         mo = new VectorMapOperator();
       } else {



Mime
View raw message