hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sunc...@apache.org
Subject svn commit: r1665637 [1/2] - in /hive/branches/parquet: ./ beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/ beeline/src/test/org/apache/hive/beeline/ beeline/src/test/resources/ common/src/java/org/apache/hadoop/hive/conf/ dev-supp...
Date Tue, 10 Mar 2015 17:49:41 GMT
Author: sunchao
Date: Tue Mar 10 17:49:39 2015
New Revision: 1665637

URL: http://svn.apache.org/r1665637
Log:
Merge trunk into parquet branch

Added:
    hive/branches/parquet/beeline/src/test/resources/
      - copied from r1665631, hive/trunk/beeline/src/test/resources/
    hive/branches/parquet/dev-support/jenkins-execute-hms-test.sh
      - copied unchanged from r1665631, hive/trunk/dev-support/jenkins-execute-hms-test.sh
    hive/branches/parquet/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.12.0-to-0.13.0.mssql.sql
      - copied unchanged from r1665631, hive/trunk/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.12.0-to-0.13.0.mssql.sql
    hive/branches/parquet/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.13.0-to-0.14.0.mssql.sql
      - copied unchanged from r1665631, hive/trunk/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.13.0-to-0.14.0.mssql.sql
    hive/branches/parquet/ql/src/test/queries/clientpositive/tez_join.q
      - copied unchanged from r1665631, hive/trunk/ql/src/test/queries/clientpositive/tez_join.q
    hive/branches/parquet/ql/src/test/results/clientpositive/tez/tez_join.q.out
      - copied unchanged from r1665631, hive/trunk/ql/src/test/results/clientpositive/tez/tez_join.q.out
    hive/branches/parquet/testutils/metastore/
      - copied from r1665631, hive/trunk/testutils/metastore/
Modified:
    hive/branches/parquet/   (props changed)
    hive/branches/parquet/LICENSE
    hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java
    hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java
    hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java
    hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java
    hive/branches/parquet/beeline/src/main/resources/BeeLine.properties
    hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
    hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml
    hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java
    hive/branches/parquet/itests/src/test/resources/testconfiguration.properties
    hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql
    hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql
    hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql
    hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
    hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    hive/branches/parquet/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
    hive/branches/parquet/ql/src/test/results/clientpositive/auto_join11.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/auto_join12.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/auto_join13.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/auto_join27.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/auto_join4.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/auto_join5.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/auto_join8.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/constantPropagateForSubQuery.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/correlationoptimizer1.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/explain_logical.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/metadataonly1.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/ppd_join_filter.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/ppd_vc.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_1.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/subquery_in_having.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/subquery_notin.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
    hive/branches/parquet/ql/src/test/results/clientpositive/table_access_keys_stats.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/tez/mrr.q.out
    hive/branches/parquet/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
    hive/branches/parquet/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java
    hive/branches/parquet/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
    hive/branches/parquet/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java

Propchange: hive/branches/parquet/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 10 17:49:39 2015
@@ -4,4 +4,4 @@
 /hive/branches/spark:1608589-1660298
 /hive/branches/tez:1494760-1622766
 /hive/branches/vectorization:1466908-1527856
-/hive/trunk:1656617-1664450
+/hive/trunk:1656617-1665631

Modified: hive/branches/parquet/LICENSE
URL: http://svn.apache.org/viewvc/hive/branches/parquet/LICENSE?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/LICENSE (original)
+++ hive/branches/parquet/LICENSE Tue Mar 10 17:49:39 2015
@@ -465,4 +465,33 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTI
 AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
-OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
+OF THE POSSIBILITY OF SUCH DAMAGE.
+
+For the PostgreSQL JDBC driver jar file:
+
+Copyright (c) 1997-2011, PostgreSQL Global Development Group
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+3. Neither the name of the PostgreSQL Global Development Group nor the names
+   of its contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file

Modified: hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java (original)
+++ hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java Tue Mar 10 17:49:39 2015
@@ -28,11 +28,9 @@ import java.io.Closeable;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
-import java.io.PrintWriter;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
@@ -53,6 +51,7 @@ import java.sql.Statement;
 import java.text.ChoiceFormat;
 import java.text.MessageFormat;
 import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -82,7 +81,6 @@ import jline.console.ConsoleReader;
 import jline.console.history.History;
 import jline.console.history.FileHistory;
 
-import jline.internal.Log;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.OptionBuilder;
@@ -157,6 +155,8 @@ public class BeeLine implements Closeabl
       "xmlelements", new XMLElementOutputFormat(this),
   });
 
+  private List<String> supportedLocalDriver =
+    new ArrayList<String>(Arrays.asList("com.mysql.jdbc.Driver", "org.postgresql.Driver"));
 
   final CommandHandler[] commandHandlers = new CommandHandler[] {
       new ReflectiveCommandHandler(this, new String[] {"quit", "done", "exit"},
@@ -249,6 +249,10 @@ public class BeeLine implements Closeabl
           null),
       new ReflectiveCommandHandler(this, new String[] {"nullemptystring"},
           new Completer[] {new BooleanCompleter()}),
+      new ReflectiveCommandHandler(this, new String[]{"addlocaldriverjar"},
+          null),
+      new ReflectiveCommandHandler(this, new String[]{"addlocaldrivername"},
+          null)
   };
 
 
@@ -1579,6 +1583,11 @@ public class BeeLine implements Closeabl
         return true;
       }
 
+      // find whether exists a local driver to accept the url
+      if (findLocalDriver(url) != null) {
+        return true;
+      }
+
       return false;
     } catch (Exception e) {
       debug(e.toString());
@@ -1601,6 +1610,40 @@ public class BeeLine implements Closeabl
     return null;
   }
 
+  public Driver findLocalDriver(String url) throws Exception {
+    if(drivers == null){
+      return null;
+    }
+
+    for (Driver d : drivers) {
+      try {
+        String clazzName = d.getClass().getName();
+        Driver driver = (Driver) Class.forName(clazzName, true,
+          Thread.currentThread().getContextClassLoader()).newInstance();
+        if (driver.acceptsURL(url) && isSupportedLocalDriver(driver)) {
+          return driver;
+        }
+      } catch (SQLException e) {
+        error(e);
+        throw new Exception(e);
+      }
+    }
+    return null;
+  }
+
+  public boolean isSupportedLocalDriver(Driver driver) {
+    String driverName = driver.getClass().getName();
+    for (String name : supportedLocalDriver) {
+      if (name.equals(driverName)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public void addLocalDriverClazz(String driverClazz) {
+    supportedLocalDriver.add(driverClazz);
+  }
 
   Driver[] scanDrivers(String line) throws IOException {
     return scanDrivers(false);

Modified: hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java (original)
+++ hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java Tue Mar 10 17:49:39 2015
@@ -52,9 +52,12 @@
 package org.apache.hive.beeline;
 
 import jline.console.completer.StringsCompleter;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.FileInputStream;
 import java.net.JarURLConnection;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -62,11 +65,15 @@ import java.net.URLConnection;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
 import java.util.Set;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 import java.util.Enumeration;
 import java.util.TreeSet;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipEntry;
 
 /**
  * the completer is original provided in JLine 0.9.94 and is being removed in 2.12. Add the
@@ -74,6 +81,10 @@ import java.util.TreeSet;
  */
 public class ClassNameCompleter extends StringsCompleter {
 
+  private static final Log LOG = LogFactory.getLog(ClassNameCompleter.class.getName());
+  public final static String clazzFileNameExtension = ".class";
+  public final static String jarFileNameExtension = ".jar";
+
   public ClassNameCompleter(String... candidates) {
     super(candidates);
   }
@@ -81,7 +92,7 @@ public class ClassNameCompleter extends
   public static String[] getClassNames() throws IOException {
     Set urls = new HashSet();
 
-    for (ClassLoader loader = ClassNameCompleter.class.getClassLoader(); loader != null;
+    for (ClassLoader loader = Thread.currentThread().getContextClassLoader(); loader != null;
          loader = loader.getParent()) {
       if (!(loader instanceof URLClassLoader)) {
         continue;
@@ -97,7 +108,7 @@ public class ClassNameCompleter extends
 
     for (int i = 0; i < systemClasses.length; i++) {
       URL classURL = systemClasses[i]
-              .getResource("/" + systemClasses[i].getName().replace('.', '/') + ".class");
+              .getResource("/" + systemClasses[i].getName().replace('.', '/') + clazzFileNameExtension);
 
       if (classURL != null) {
         URLConnection uc = classURL.openConnection();
@@ -136,12 +147,14 @@ public class ClassNameCompleter extends
 
         String name = entry.getName();
 
-        if (!name.endsWith(".class")) {
+        if (isClazzFile(name)) {
           /* only use class file*/
+          classes.add(name);
+        } else if (isJarFile(name)) {
+          classes.addAll(getClassNamesFromJar(name));
+        } else {
           continue;
         }
-
-        classes.add(name);
       }
     }
 
@@ -151,8 +164,7 @@ public class ClassNameCompleter extends
 
     for (Iterator i = classes.iterator(); i.hasNext(); ) {
       String name = (String) i.next();
-      classNames.add(name.replace('/', '.').
-              substring(0, name.length() - 6));
+      classNames.add(name.replace('/', '.').substring(0, name.length() - 6));
     }
 
     return (String[]) classNames.toArray(new String[classNames.size()]);
@@ -173,7 +185,7 @@ public class ClassNameCompleter extends
         continue;
       } else if (files[i].isDirectory()) {
         getClassFiles(root, holder, files[i], maxDirectories);
-      } else if (files[i].getName().endsWith(".class")) {
+      } else if (files[i].getName().endsWith(clazzFileNameExtension)) {
         holder.add(files[i].getAbsolutePath().
                 substring(root.length() + 1));
       }
@@ -181,4 +193,54 @@ public class ClassNameCompleter extends
 
     return holder;
   }
+
+  /**
+   * Get clazz names from a jar file path
+   * @param path specifies the jar file's path
+   * @return
+   */
+  private static List<String> getClassNamesFromJar(String path) {
+    List<String> classNames = new ArrayList<String>();
+    ZipInputStream zip = null;
+    try {
+      zip = new ZipInputStream(new FileInputStream(path));
+      ZipEntry entry = zip.getNextEntry();
+      while (entry != null) {
+        if (!entry.isDirectory() && entry.getName().endsWith(clazzFileNameExtension)) {
+          StringBuilder className = new StringBuilder();
+          for (String part : entry.getName().split("/")) {
+            if (className.length() != 0) {
+              className.append(".");
+            }
+            className.append(part);
+            if (part.endsWith(clazzFileNameExtension)) {
+              className.setLength(className.length() - clazzFileNameExtension.length());
+            }
+          }
+          classNames.add(className.toString());
+        }
+        entry = zip.getNextEntry();
+      }
+    } catch (IOException e) {
+      LOG.error("Fail to parse the class name from the Jar file due to the exception:" + e);
+    } finally {
+      if (zip != null) {
+        try {
+          zip.close();
+        } catch (IOException e) {
+          LOG.error("Fail to close the file due to the exception:" + e);
+        }
+      }
+    }
+
+    return classNames;
+  }
+
+  private static boolean isJarFile(String fileName) {
+    return fileName.endsWith(jarFileNameExtension);
+  }
+
+  private static boolean isClazzFile(String clazzName) {
+    return clazzName.endsWith(clazzFileNameExtension);
+  }
 }

Modified: hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java (original)
+++ hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java Tue Mar 10 17:49:39 2015
@@ -25,12 +25,16 @@ package org.apache.hive.beeline;
 import org.apache.hadoop.io.IOUtils;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.sql.CallableStatement;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -126,6 +130,46 @@ public class Commands {
     return true;
   }
 
+  public boolean addlocaldrivername(String line) {
+    String driverName = arg1(line, "driver class name");
+    try {
+      beeLine.setDrivers(Arrays.asList(beeLine.scanDrivers(false)));
+    } catch (IOException e) {
+      beeLine.error("Fail to scan drivers due to the exception:" + e);
+      beeLine.error(e);
+    }
+    for (Driver d : beeLine.getDrivers()) {
+      if (driverName.equals(d.getClass().getName())) {
+        beeLine.addLocalDriverClazz(driverName);
+        return true;
+      }
+    }
+    beeLine.error("Fail to find a driver which contains the driver class");
+    return false;
+  }
+
+  public boolean addlocaldriverjar(String line) {
+    // If jar file is in the hdfs, it should be downloaded first.
+    String jarPath = arg1(line, "jar path");
+    File p = new File(jarPath);
+    if (!p.exists()) {
+      beeLine.error("The jar file in the path " + jarPath + " can't be found!");
+      return false;
+    }
+
+    URLClassLoader classLoader = (URLClassLoader) Thread.currentThread().getContextClassLoader();
+    try {
+      beeLine.debug(jarPath + " is added to the local beeline.");
+      URLClassLoader newClassLoader = new URLClassLoader(new URL[]{p.toURL()}, classLoader);
+
+      Thread.currentThread().setContextClassLoader(newClassLoader);
+      beeLine.setDrivers(Arrays.asList(beeLine.scanDrivers(false)));
+    } catch (Exception e) {
+      beeLine.error("Fail to add local jar due to the exception:" + e);
+      beeLine.error(e);
+    }
+    return true;
+  }
 
   public boolean history(String line) {
     Iterator hist = beeLine.getConsoleReader().getHistory().entries();

Modified: hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java (original)
+++ hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java Tue Mar 10 17:49:39 2015
@@ -25,9 +25,12 @@ package org.apache.hive.beeline;
 import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
+import java.sql.Driver;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -112,9 +115,9 @@ class DatabaseConnection {
       return beeLine.error(cnfe);
     }
 
-    boolean foundDriver = false;
+    boolean isDriverRegistered = false;
     try {
-      foundDriver = DriverManager.getDriver(getUrl()) != null;
+      isDriverRegistered = DriverManager.getDriver(getUrl()) != null;
     } catch (Exception e) {
     }
 
@@ -134,7 +137,13 @@ class DatabaseConnection {
       info.put(HIVE_CONF_PREFIX + var.getKey(), var.getValue());
     }
 
-    setConnection(DriverManager.getConnection(getUrl(), info));
+    if (isDriverRegistered) {
+      // if the driver registered in the driver manager, get the connection via the driver manager
+      setConnection(DriverManager.getConnection(getUrl(), info));
+    } else {
+      beeLine.debug("Use the driver from local added jar file.");
+      setConnection(getConnectionFromLocalDriver(getUrl(), info));
+    }
     setDatabaseMetaData(getConnection().getMetaData());
 
     try {
@@ -170,6 +179,26 @@ class DatabaseConnection {
     return true;
   }
 
+  public Connection getConnectionFromLocalDriver(String url, Properties properties) {
+    Collection<Driver> drivers = beeLine.getDrivers();
+    for (Driver d : drivers) {
+      try {
+        if (d.acceptsURL(url) && beeLine.isSupportedLocalDriver(d)) {
+          String clazzName = d.getClass().getName();
+          beeLine.debug("Driver name is " + clazzName);
+          Driver driver =
+            (Driver) Class.forName(clazzName, true, Thread.currentThread().getContextClassLoader())
+              .newInstance();
+          return driver.connect(url, properties);
+        }
+      } catch (Exception e) {
+        beeLine.error("Fail to connect with a local driver due to the exception:" + e);
+        beeLine.error(e);
+      }
+    }
+    return null;
+  }
+
 
   public Connection getConnection() throws SQLException {
     if (connection != null) {

Modified: hive/branches/parquet/beeline/src/main/resources/BeeLine.properties
URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/main/resources/BeeLine.properties?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/beeline/src/main/resources/BeeLine.properties (original)
+++ hive/branches/parquet/beeline/src/main/resources/BeeLine.properties Tue Mar 10 17:49:39 2015
@@ -71,6 +71,8 @@ help-properties: Connect to the database
 help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv))
 help-delimiterForDSV: Set the delimiter for dsv output format
 help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false.
+help-addlocaldriverjar: Add driver jar file in the beeline client side.
+help-addlocaldrivername: Add driver name that needs to be supported in the beeline client side.
 
 jline-missing: The JLine jar was not found. Please ensure it is installed.
 
@@ -174,6 +176,8 @@ cmd-usage: Usage: java org.apache.hive.c
 \  --delimiterForDSV=DELIMITER     specify the delimiter for delimiter-separated values output format (default: |)\n \
 \  --isolation=LEVEL               set the transaction isolation level\n \
 \  --nullemptystring=[true/false]  set to true to get historic behavior of printing null as empty string\n \
+\  --addlocaldriverjar=DRIVERJARNAME Add driver jar file in the beeline client side\n \
+\  --addlocaldrivername=DRIVERNAME Add drvier name needs to be supported in the beeline client side\n \
 \  --help                          display this message
 
 

Modified: hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java (original)
+++ hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java Tue Mar 10 17:49:39 2015
@@ -19,19 +19,40 @@
 package org.apache.hive.beeline;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 
 import java.io.File;
 import java.io.FileOutputStream;
 
-import junit.framework.Assert;
-
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hive.common.util.HiveTestUtils;
+import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
 
 /**
  * Unit test for Beeline arg parser.
  */
+@RunWith(Parameterized.class)
 public class TestBeelineArgParsing {
+  private static final Log LOG = LogFactory.getLog(TestBeelineArgParsing.class.getName());
+  private String connectionString;
+  private String driverClazzName;
+  private String driverJarFileName;
+  private boolean defaultSupported;
+
+  public TestBeelineArgParsing(String connectionString, String driverClazzName, String driverJarFileName,
+                               boolean defaultSupported) {
+    this.connectionString = connectionString;
+    this.driverClazzName = driverClazzName;
+    this.driverJarFileName = driverJarFileName;
+    this.defaultSupported = defaultSupported;
+  }
 
   public class TestBeeline extends BeeLine {
 
@@ -52,6 +73,27 @@ public class TestBeelineArgParsing {
       }
       return true;
     }
+
+    public boolean addlocaldrivername(String driverName) {
+      String line = "addlocaldrivername " + driverName;
+      return getCommands().addlocaldrivername(line);
+    }
+
+    public boolean addLocalJar(String url){
+      String line = "addlocaldriverjar " + url;
+      return getCommands().addlocaldriverjar(line);
+    }
+  }
+
+  @Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(
+        new Object[][] {
+            {"jdbc:postgresql://host:5432/testdb", "org.postgresql.Driver", "postgresql-9.3.jdbc3.jar",
+                true},
+            {"jdbc:dummy://host:5432/testdb", "org.apache.dummy.DummyDriver",
+                "DummyDriver-1.0-SNAPSHOT.jar", false}
+        });
   }
 
   @Test
@@ -59,7 +101,7 @@ public class TestBeelineArgParsing {
     TestBeeline bl = new TestBeeline();
     String args[] = new String[] {"-u", "url", "-n", "name",
       "-p", "password", "-d", "driver", "-a", "authType"};
-    Assert.assertEquals(0, bl.initArgs(args));
+    org.junit.Assert.assertEquals(0, bl.initArgs(args));
     Assert.assertTrue(bl.connectArgs.equals("url name password driver"));
     Assert.assertTrue(bl.getOpts().getAuthType().equals("authType"));
   }
@@ -169,4 +211,32 @@ public class TestBeelineArgParsing {
     Assert.assertEquals(-1, bl.initArgs(args));
   }
 
+  @Test
+  public void testAddLocalJar() throws Exception {
+    TestBeeline bl = new TestBeeline();
+    Assert.assertNull(bl.findLocalDriver(connectionString));
+
+    LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName);
+    String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName);
+
+    bl.addLocalJar(mysqlDriverPath);
+    bl.addlocaldrivername(driverClazzName);
+    Assert.assertEquals(bl.findLocalDriver(connectionString).getClass().getName(), driverClazzName);
+  }
+
+  @Test
+  public void testAddLocalJarWithoutAddDriverClazz() throws Exception {
+    TestBeeline bl = new TestBeeline();
+
+    LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName);
+    String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName);
+
+    bl.addLocalJar(mysqlDriverPath);
+    if (!defaultSupported) {
+      Assert.assertNull(bl.findLocalDriver(connectionString));
+    } else {
+      // no need to add for the default supported local jar driver
+      Assert.assertEquals(bl.findLocalDriver(connectionString).getClass().getName(), driverClazzName);
+    }
+  }
 }

Modified: hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Mar 10 17:49:39 2015
@@ -601,6 +601,9 @@ public class HiveConf extends Configurat
         + "is set to instance of HiveAuthorizerFactory, then this value is ignored."),
     FIRE_EVENTS_FOR_DML("hive.metastore.dml.events", false, "If true, the metastore will be asked" +
         " to fire events for DML operations"),
+    METASTORE_CLIENT_DROP_PARTITIONS_WITH_EXPRESSIONS("hive.metastore.client.drop.partitions.using.expressions", true,
+        "Choose whether dropping partitions with HCatClient pushes the partition-predicate to the metastore, " +
+            "or drops partitions iteratively"),
 
     // Parameters for exporting metadata on table drop (requires the use of the)
     // org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener

Modified: hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml (original)
+++ hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml Tue Mar 10 17:49:39 2015
@@ -45,6 +45,11 @@
       <artifactId>hive-hcatalog-core</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <!-- test intra-project -->
     <dependency>
       <groupId>org.apache.hive</groupId>

Modified: hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java (original)
+++ hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java Tue Mar 10 17:49:39 2015
@@ -20,12 +20,15 @@ package org.apache.hive.hcatalog.api;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.ObjectPair;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
 import org.apache.hadoop.hive.common.classification.InterfaceStability;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -36,6 +39,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
 import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -48,12 +52,25 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.UnknownDBException;
 import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hive.hcatalog.common.HCatConstants;
 import org.apache.hive.hcatalog.common.HCatException;
 import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils;
 import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The HCatClientHMSImpl is the Hive Metastore client based implementation of
@@ -61,6 +78,7 @@ import org.apache.thrift.TException;
  */
 public class HCatClientHMSImpl extends HCatClient {
 
+  private static final Logger LOG = LoggerFactory.getLogger(HCatClientHMSImpl.class);
   private HiveMetaStoreClient hmsClient;
   private Configuration config;
   private HiveConf hiveConfig;
@@ -480,19 +498,122 @@ public class HCatClientHMSImpl extends H
     }
   }
 
+  /**
+   * Helper class to help build ExprDesc tree to represent the partitions to be dropped.
+   * Note: At present, the ExpressionBuilder only constructs partition predicates where
+   * partition-keys equal specific values, and logical-AND expressions. E.g.
+   *  ( dt = '20150310' AND region = 'US' )
+   * This only supports the partition-specs specified by the Map argument of:
+   * {@link org.apache.hive.hcatalog.api.HCatClient#dropPartitions(String, String, Map, boolean)}
+   */
+  private static class ExpressionBuilder {
+
+    private Map<String, PrimitiveTypeInfo> partColumnTypesMap = Maps.newHashMap();
+    private Map<String, String> partSpecs;
+
+    public ExpressionBuilder(Table table, Map<String, String> partSpecs) {
+      this.partSpecs = partSpecs;
+      for (FieldSchema partField : table.getPartitionKeys()) {
+        partColumnTypesMap.put(partField.getName().toLowerCase(),
+            TypeInfoFactory.getPrimitiveTypeInfo(partField.getType()));
+      }
+    }
+
+    private PrimitiveTypeInfo getTypeFor(String partColumn) {
+      return partColumnTypesMap.get(partColumn.toLowerCase());
+    }
+
+    private Object getTypeAppropriateValueFor(PrimitiveTypeInfo type, String value) {
+      ObjectInspectorConverters.Converter converter = ObjectInspectorConverters.getConverter(
+          TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(TypeInfoFactory.stringTypeInfo),
+          TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(type));
+
+      return converter.convert(value);
+    }
+
+    public ExprNodeGenericFuncDesc equalityPredicate(String partColumn, String value) throws SemanticException {
+
+      PrimitiveTypeInfo partColumnType = getTypeFor(partColumn);
+      ExprNodeColumnDesc partColumnExpr = new ExprNodeColumnDesc(partColumnType, partColumn, null, true);
+      ExprNodeConstantDesc valueExpr = new ExprNodeConstantDesc(partColumnType,
+          getTypeAppropriateValueFor(partColumnType, value));
+
+      return binaryPredicate("=", partColumnExpr, valueExpr);
+    }
+
+    public ExprNodeGenericFuncDesc binaryPredicate(String function, ExprNodeDesc lhs, ExprNodeDesc rhs) throws SemanticException {
+      return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
+          FunctionRegistry.getFunctionInfo(function).getGenericUDF(),
+          Lists.newArrayList(lhs, rhs));
+    }
+
+    public ExprNodeGenericFuncDesc build() throws SemanticException {
+      ExprNodeGenericFuncDesc resultExpr = null;
+
+      for (Map.Entry<String,String> partSpec : partSpecs.entrySet()) {
+        String column = partSpec.getKey();
+        String value  = partSpec.getValue();
+        ExprNodeGenericFuncDesc partExpr = equalityPredicate(column, value);
+
+        resultExpr = (resultExpr == null? partExpr : binaryPredicate("and", resultExpr, partExpr));
+      }
+
+      return resultExpr;
+    }
+  } // class ExpressionBuilder;
+
+  private static boolean isExternal(Table table) {
+    return table.getParameters() != null
+        && "TRUE".equalsIgnoreCase(table.getParameters().get("EXTERNAL"));
+  }
+
+  private void dropPartitionsUsingExpressions(Table table, Map<String, String> partitionSpec, boolean ifExists)
+    throws SemanticException, TException {
+    LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions.");
+    ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build();
+    ObjectPair<Integer, byte[]> serializedPartitionExpression =
+        new ObjectPair<Integer, byte[]>(partitionSpec.size(),
+            Utilities.serializeExpressionToKryo(partitionExpression));
+    hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression),
+        !isExternal(table),  // Delete data?
+        false,               // Ignore Protection?
+        ifExists,            // Fail if table doesn't exist?
+        false);              // Need results back?
+  }
+
+  private void dropPartitionsIteratively(String dbName, String tableName,
+                                         Map<String, String> partitionSpec, boolean ifExists) throws HCatException, TException {
+    LOG.info("HCatClient: Dropping partitions iteratively.");
+    List<Partition> partitions = hmsClient.listPartitionsByFilter(dbName, tableName,
+        getFilterString(partitionSpec), (short) -1);
+    for (Partition partition : partitions) {
+      dropPartition(partition, ifExists);
+    }
+  }
+
   @Override
   public void dropPartitions(String dbName, String tableName,
                  Map<String, String> partitionSpec, boolean ifExists)
     throws HCatException {
+    LOG.info("HCatClient dropPartitions(db=" + dbName + ",table=" + tableName + ", partitionSpec: ["+ partitionSpec + "]).");
     try {
       dbName = checkDB(dbName);
-      List<Partition> partitions = hmsClient.listPartitionsByFilter(dbName, tableName,
-          getFilterString(partitionSpec), (short)-1);
+      Table table = hmsClient.getTable(dbName, tableName);
 
-      for (Partition partition : partitions) {
-        dropPartition(partition, ifExists);
+      if (hiveConfig.getBoolVar(HiveConf.ConfVars.METASTORE_CLIENT_DROP_PARTITIONS_WITH_EXPRESSIONS)) {
+        try {
+          dropPartitionsUsingExpressions(table, partitionSpec, ifExists);
+        }
+        catch (SemanticException parseFailure) {
+          LOG.warn("Could not push down partition-specification to back-end, for dropPartitions(). Resorting to iteration.",
+              parseFailure);
+          dropPartitionsIteratively(dbName, tableName, partitionSpec, ifExists);
+        }
+      }
+      else {
+        // Not using expressions.
+        dropPartitionsIteratively(dbName, tableName, partitionSpec, ifExists);
       }
-
     } catch (NoSuchObjectException e) {
       throw new ObjectNotFoundException(
           "NoSuchObjectException while dropping partition. " +

Modified: hive/branches/parquet/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/parquet/itests/src/test/resources/testconfiguration.properties?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/parquet/itests/src/test/resources/testconfiguration.properties Tue Mar 10 17:49:39 2015
@@ -305,7 +305,8 @@ minitez.query.files=bucket_map_join_tez1
   tez_smb_main.q,\
   tez_smb_1.q,\
   vectorized_dynamic_partition_pruning.q,\
-  tez_multi_union.q
+  tez_multi_union.q,\
+  tez_join.q
 
 encrypted.query.files=encryption_join_unencrypted_tbl.q,\
   encryption_insert_partition_static.q,\

Modified: hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql
URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql (original)
+++ hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql Tue Mar 10 17:49:39 2015
@@ -1,32 +1 @@
---
--- Create the table if it doesn't exist.
---
-if not exists (SELECT 1 FROM INFORMATION_SCHEMA.TABLES 
-           WHERE TABLE_NAME='PART_COL_STATS')
-CREATE TABLE PART_COL_STATS
-(
-    CS_ID bigint NOT NULL,
-    AVG_COL_LEN float NULL,
-    "COLUMN_NAME" varchar(128) NOT NULL,
-    COLUMN_TYPE varchar(128) NOT NULL,
-    DB_NAME varchar(128) NOT NULL,
-    BIG_DECIMAL_HIGH_VALUE varchar(255) NULL,
-    BIG_DECIMAL_LOW_VALUE varchar(255) NULL,
-    DOUBLE_HIGH_VALUE float NULL,
-    DOUBLE_LOW_VALUE float NULL,
-    LAST_ANALYZED bigint NOT NULL,
-    LONG_HIGH_VALUE bigint NULL,
-    LONG_LOW_VALUE bigint NULL,
-    MAX_COL_LEN bigint NULL,
-    NUM_DISTINCTS bigint NULL,
-    NUM_FALSES bigint NULL,
-    NUM_NULLS bigint NOT NULL,
-    NUM_TRUES bigint NULL,
-    PART_ID bigint NULL,
-    PARTITION_NAME varchar(767) NOT NULL,
-    "TABLE_NAME" varchar(128) NOT NULL
-)
-go
-
-
 CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (DB_NAME,TABLE_NAME,COLUMN_NAME,PARTITION_NAME);

Modified: hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql
URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql (original)
+++ hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql Tue Mar 10 17:49:39 2015
@@ -2,12 +2,9 @@
 DROP INDEX PARTITIONS.UNIQUEPARTITION;
 ALTER TABLE PARTITIONS ALTER COLUMN PART_NAME NVARCHAR(767) NULL;
 CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (PART_NAME,TBL_ID);
-GO
 
 --ALTER SDS.LOCATION from varchar to nvarchar
 ALTER TABLE SDS ALTER COLUMN LOCATION NVARCHAR(4000) NULL;
-GO
 
 --ALTER PARTITION_KEY_VALS.PART_KEY_VAL from varchar to nvarchar
 ALTER TABLE PARTITION_KEY_VALS ALTER COLUMN PART_KEY_VAL NVARCHAR(255) NULL;
-GO
\ No newline at end of file

Modified: hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql
URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql (original)
+++ hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql Tue Mar 10 17:49:39 2015
@@ -8,19 +8,18 @@ CREATE TABLE NOTIFICATION_LOG
     TBL_NAME varchar(128) NULL,
     MESSAGE text NULL
 );
-GO
+
 
 ALTER TABLE NOTIFICATION_LOG ADD CONSTRAINT NOTIFICATION_LOG_PK PRIMARY KEY (NL_ID);
-GO
 
 CREATE TABLE NOTIFICATION_SEQUENCE
 (
     NNI_ID bigint NOT NULL,
     NEXT_EVENT_ID bigint NOT NULL
 );
-GO
+
 
 ALTER TABLE NOTIFICATION_SEQUENCE ADD CONSTRAINT NOTIFICATION_SEQUENCE_PK PRIMARY KEY (NNI_ID);
-GO
+
 
 

Modified: hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Tue Mar 10 17:49:39 2015
@@ -4142,12 +4142,16 @@ public class HiveMetaStore extends Thrif
     @Override
     public TableStatsResult get_table_statistics_req(TableStatsRequest request)
         throws MetaException, NoSuchObjectException, TException {
-      String dbName = request.getDbName(), tblName = request.getTblName();
+      String dbName = request.getDbName().toLowerCase();
+      String tblName = request.getTblName().toLowerCase();
       startFunction("get_table_statistics_req: db=" + dbName + " table=" + tblName);
       TableStatsResult result = null;
+      List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size());
+      for (String colName : request.getColNames()) {
+        lowerCaseColNames.add(colName.toLowerCase());
+      }
       try {
-        ColumnStatistics cs = getMS().getTableColumnStatistics(
-            dbName, tblName, request.getColNames());
+        ColumnStatistics cs = getMS().getTableColumnStatistics(dbName, tblName, lowerCaseColNames);
         result = new TableStatsResult(
             cs == null ? Lists.<ColumnStatisticsObj>newArrayList() : cs.getStatsObj());
       } finally {
@@ -4185,13 +4189,22 @@ public class HiveMetaStore extends Thrif
     @Override
     public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsRequest request)
         throws MetaException, NoSuchObjectException, TException {
-      String dbName = request.getDbName(), tblName = request.getTblName();
+      String dbName = request.getDbName().toLowerCase();
+      String tblName = request.getTblName().toLowerCase();
       startFunction("get_partitions_statistics_req: db=" + dbName + " table=" + tblName);
 
       PartitionsStatsResult result = null;
+      List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size());
+      for (String colName : request.getColNames()) {
+        lowerCaseColNames.add(colName.toLowerCase());
+      }
+      List<String> lowerCasePartNames = new ArrayList<String>(request.getPartNames().size());
+      for (String partName : request.getPartNames()) {
+        lowerCasePartNames.add(lowerCaseConvertPartName(partName));
+      }
       try {
         List<ColumnStatistics> stats = getMS().getPartitionColumnStatistics(
-            dbName, tblName, request.getPartNames(), request.getColNames());
+            dbName, tblName, lowerCasePartNames, lowerCaseColNames);
         Map<String, List<ColumnStatisticsObj>> map =
             new HashMap<String, List<ColumnStatisticsObj>>();
         for (ColumnStatistics stat : stats) {
@@ -5579,11 +5592,23 @@ public class HiveMetaStore extends Thrif
     @Override
     public AggrStats get_aggr_stats_for(PartitionsStatsRequest request)
         throws NoSuchObjectException, MetaException, TException {
+      String dbName = request.getDbName().toLowerCase();
+      String tblName = request.getTblName().toLowerCase();
       startFunction("get_aggr_stats_for: db=" + request.getDbName() + " table=" + request.getTblName());
+
+      List<String> lowerCaseColNames = new ArrayList<String>(request.getColNames().size());
+      for (String colName : request.getColNames()) {
+        lowerCaseColNames.add(colName.toLowerCase());
+      }
+      List<String> lowerCasePartNames = new ArrayList<String>(request.getPartNames().size());
+      for (String partName : request.getPartNames()) {
+        lowerCasePartNames.add(lowerCaseConvertPartName(partName));
+      }
       AggrStats aggrStats = null;
+
       try {
-        aggrStats = new AggrStats(getMS().get_aggr_stats_for(request.getDbName(),
-          request.getTblName(), request.getPartNames(), request.getColNames()));
+        aggrStats = new AggrStats(getMS().get_aggr_stats_for(dbName, tblName, lowerCasePartNames,
+            lowerCaseColNames));
         return aggrStats;
       } finally {
           endFunction("get_partitions_statistics_req: ", aggrStats == null, null, request.getTblName());

Modified: hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Tue Mar 10 17:49:39 2015
@@ -32,9 +32,7 @@ import java.util.Map.Entry;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import javax.security.auth.login.LoginException;
 
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -55,8 +53,6 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 
 /**
@@ -250,41 +246,25 @@ public class Warehouse {
       return false;
     }
     final FileStatus stat;
+    final FileSystem fs;
     try {
-      stat = getFs(path).getFileStatus(path);
+      fs = getFs(path);
+      stat = fs.getFileStatus(path);
+      ShimLoader.getHadoopShims().checkFileAccess(fs, stat, FsAction.WRITE);
+      return true;
     } catch (FileNotFoundException fnfe){
       // File named by path doesn't exist; nothing to validate.
       return true;
     } catch (Exception e) {
       // all other exceptions are considered as emanating from
       // unauthorized accesses
-      return false;
-    }
-    final UserGroupInformation ugi;
-    try {
-      ugi = Utils.getUGI();
-    } catch (LoginException le) {
-      throw new IOException(le);
-    }
-    String user = ugi.getShortUserName();
-    //check whether owner can delete
-    if (stat.getOwner().equals(user) &&
-        stat.getPermission().getUserAction().implies(FsAction.WRITE)) {
-      return true;
-    }
-    //check whether group of the user can delete
-    if (stat.getPermission().getGroupAction().implies(FsAction.WRITE)) {
-      String[] groups = ugi.getGroupNames();
-      if (ArrayUtils.contains(groups, stat.getGroup())) {
-        return true;
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Exception when checking if path (" + path + ")", e);
       }
+      return false;
     }
-    //check whether others can delete (uncommon case!!)
-    if (stat.getPermission().getOtherAction().implies(FsAction.WRITE)) {
-      return true;
-    }
-    return false;
   }
+
   /*
   // NOTE: This is for generating the internal path name for partitions. Users
   // should always use the MetaStore API to get the path name for a partition.

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java Tue Mar 10 17:49:39 2015
@@ -38,6 +38,7 @@ import org.apache.hadoop.security.token.
  */
 public class SecureCmdDoAs {
   private final Path tokenPath;
+  private final File tokenFile;
 
   public SecureCmdDoAs(HiveConf conf) throws HiveException, IOException{
     // Get delegation token for user from filesystem and write the token along with
@@ -46,8 +47,8 @@ public class SecureCmdDoAs {
     FileSystem fs = FileSystem.get(conf);
     Token<?> fsToken = fs.getDelegationToken(uname);
 
-    File t = File.createTempFile("hive_hadoop_delegation_token", null);
-    tokenPath = new Path(t.toURI());
+    tokenFile = File.createTempFile("hive_hadoop_delegation_token", null);
+    tokenPath = new Path(tokenFile.toURI());
 
     //write credential with token to file
     Credentials cred = new Credentials();
@@ -60,4 +61,7 @@ public class SecureCmdDoAs {
         tokenPath.toUri().getPath());
   }
 
+  public void close() {
+    tokenFile.delete();
+  }
 }

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java Tue Mar 10 17:49:39 2015
@@ -95,6 +95,7 @@ public class MapredLocalTask extends Tas
   private ExecMapperContext execContext = null;
 
   private Process executor;
+  private SecureCmdDoAs secureDoAs;
 
   public MapredLocalTask() {
     super();
@@ -269,7 +270,7 @@ public class MapredLocalTask extends Tas
         //If kerberos security is enabled, and HS2 doAs is enabled,
         // then additional params need to be set so that the command is run as
         // intended user
-        SecureCmdDoAs secureDoAs = new SecureCmdDoAs(conf);
+        secureDoAs = new SecureCmdDoAs(conf);
         secureDoAs.addEnv(variables);
       }
 
@@ -312,9 +313,12 @@ public class MapredLocalTask extends Tas
 
       return exitVal;
     } catch (Exception e) {
-      e.printStackTrace();
-      LOG.error("Exception: " + e.getMessage());
+      LOG.error("Exception: " + e, e);
       return (1);
+    } finally {
+      if (secureDoAs != null) {
+        secureDoAs.close();
+      }
     }
   }
 

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java Tue Mar 10 17:49:39 2015
@@ -64,7 +64,7 @@ public class ParquetRecordReaderWrapper
   private boolean eof = false;
   private int schemaSize;
   private boolean skipTimestampConversion = false;
-
+  private JobConf jobConf;
   private final ProjectionPusher projectionPusher;
 
   public ParquetRecordReaderWrapper(
@@ -86,25 +86,26 @@ public class ParquetRecordReaderWrapper
     this.splitLen = oldSplit.getLength();
     this.projectionPusher = pusher;
 
-    final ParquetInputSplit split = getSplit(oldSplit, oldJobConf);
+    jobConf = oldJobConf;
+    final ParquetInputSplit split = getSplit(oldSplit, jobConf);
 
-    TaskAttemptID taskAttemptID = TaskAttemptID.forName(oldJobConf.get(IOConstants.MAPRED_TASK_ID));
+    TaskAttemptID taskAttemptID = TaskAttemptID.forName(jobConf.get(IOConstants.MAPRED_TASK_ID));
     if (taskAttemptID == null) {
       taskAttemptID = new TaskAttemptID();
     }
 
-    setFilter(oldJobConf);
+    setFilter(jobConf);
 
     // create a TaskInputOutputContext
-    Configuration conf = oldJobConf;
+    Configuration conf = jobConf;
     if (skipTimestampConversion ^ HiveConf.getBoolVar(
         conf, HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION)) {
       conf = new JobConf(oldJobConf);
       HiveConf.setBoolVar(conf,
         HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION, skipTimestampConversion);
     }
-    final TaskAttemptContext taskContext = ContextUtil.newTaskAttemptContext(conf, taskAttemptID);
 
+    final TaskAttemptContext taskContext = ContextUtil.newTaskAttemptContext(conf, taskAttemptID);
     if (split != null) {
       try {
         realReader = newInputFormat.createRecordReader(split, taskContext);
@@ -236,14 +237,14 @@ public class ParquetRecordReaderWrapper
     ParquetInputSplit split;
     if (oldSplit instanceof FileSplit) {
       final Path finalPath = ((FileSplit) oldSplit).getPath();
-      final JobConf cloneJob = projectionPusher.pushProjectionsAndFilters(conf, finalPath.getParent());
+      jobConf = projectionPusher.pushProjectionsAndFilters(conf, finalPath.getParent());
 
-      final ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(cloneJob, finalPath);
+      final ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(jobConf, finalPath);
       final List<BlockMetaData> blocks = parquetMetadata.getBlocks();
       final FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
 
       final ReadContext readContext = new DataWritableReadSupport()
-          .init(cloneJob, fileMetaData.getKeyValueMetaData(), fileMetaData.getSchema());
+          .init(jobConf, fileMetaData.getKeyValueMetaData(), fileMetaData.getSchema());
       schemaSize = MessageTypeParser.parseMessageType(readContext.getReadSupportMetadata()
           .get(DataWritableReadSupport.HIVE_SCHEMA_KEY)).getFieldCount();
       final List<BlockMetaData> splitGroup = new ArrayList<BlockMetaData>();

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Tue Mar 10 17:49:39 2015
@@ -111,7 +111,7 @@ public class ConvertJoinMapJoin implemen
         }
 
         if (parentOp instanceof ReduceSinkOperator) {
-          ReduceSinkOperator rs = (ReduceSinkOperator)parentOp;
+          ReduceSinkOperator rs = (ReduceSinkOperator) parentOp;
           estimatedBuckets = (estimatedBuckets < rs.getConf().getNumReducers()) ?
               rs.getConf().getNumReducers() : estimatedBuckets;
         }
@@ -133,10 +133,10 @@ public class ConvertJoinMapJoin implemen
       if (retval == null) {
         return retval;
       } else {
-          // only case is full outer join with SMB enabled which is not possible. Convert to regular
-          // join.
-          convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
-          return null;
+        // only case is full outer join with SMB enabled which is not possible. Convert to regular
+        // join.
+        convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
+        return null;
       }
     }
 
@@ -160,8 +160,10 @@ public class ConvertJoinMapJoin implemen
     }
 
     MapJoinOperator mapJoinOp = convertJoinMapJoin(joinOp, context, mapJoinConversionPos);
-    // map join operator by default has no bucket cols
-    mapJoinOp.setOpTraits(new OpTraits(null, -1, null));
+    // map join operator by default has no bucket cols and num of reduce sinks
+    // reduced by 1
+    mapJoinOp
+        .setOpTraits(new OpTraits(null, -1, null, joinOp.getOpTraits().getNumReduceSinks()));
     mapJoinOp.setStatistics(joinOp.getStatistics());
     // propagate this change till the next RS
     for (Operator<? extends OperatorDesc> childOp : mapJoinOp.getChildOperators()) {
@@ -176,7 +178,8 @@ public class ConvertJoinMapJoin implemen
       TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException {
     // we cannot convert to bucket map join, we cannot convert to
     // map join either based on the size. Check if we can convert to SMB join.
-    if (context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false) {
+    if ((context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false)
+        || (joinOp.getOpTraits().getNumReduceSinks() >= 2)) {
       convertJoinSMBJoin(joinOp, context, 0, 0, false, false);
       return null;
     }
@@ -221,7 +224,7 @@ public class ConvertJoinMapJoin implemen
       convertJoinSMBJoin(joinOp, context, pos, 0, false, false);
     }
     return null;
-}
+  }
 
   // replaces the join operator with a new CommonJoinOperator, removes the
   // parent reduce sinks
@@ -240,9 +243,9 @@ public class ConvertJoinMapJoin implemen
           new MapJoinDesc(
                   MapJoinProcessor.getKeys(joinOp.getConf().isLeftInputJoin(),
                   joinOp.getConf().getBaseSrc(), joinOp).getSecond(),
-              null, joinDesc.getExprs(), null, null,
-              joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(),
-              joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null);
+                  null, joinDesc.getExprs(), null, null,
+                  joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(),
+                  joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null);
       mapJoinDesc.setNullSafes(joinDesc.getNullSafes());
       mapJoinDesc.setFilterMap(joinDesc.getFilterMap());
       mapJoinDesc.resetOrder();
@@ -251,9 +254,9 @@ public class ConvertJoinMapJoin implemen
     CommonMergeJoinOperator mergeJoinOp =
         (CommonMergeJoinOperator) OperatorFactory.get(new CommonMergeJoinDesc(numBuckets,
             isSubQuery, mapJoinConversionPos, mapJoinDesc), joinOp.getSchema());
-    OpTraits opTraits =
-        new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp.getOpTraits()
-            .getSortCols());
+    int numReduceSinks = joinOp.getOpTraits().getNumReduceSinks();
+    OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp
+        .getOpTraits().getSortCols(), numReduceSinks);
     mergeJoinOp.setOpTraits(opTraits);
     mergeJoinOp.setStatistics(joinOp.getStatistics());
 
@@ -289,8 +292,7 @@ public class ConvertJoinMapJoin implemen
 
     if (adjustParentsChildren) {
       mergeJoinOp.getConf().setGenJoinKeys(true);
-      List<Operator<? extends OperatorDesc>> newParentOpList =
-          new ArrayList<Operator<? extends OperatorDesc>>();
+      List<Operator<? extends OperatorDesc>> newParentOpList = new ArrayList<Operator<? extends OperatorDesc>>();
       for (Operator<? extends OperatorDesc> parentOp : mergeJoinOp.getParentOperators()) {
         for (Operator<? extends OperatorDesc> grandParentOp : parentOp.getParentOperators()) {
           grandParentOp.getChildOperators().remove(parentOp);
@@ -328,7 +330,8 @@ public class ConvertJoinMapJoin implemen
     if (currentOp instanceof ReduceSinkOperator) {
       return;
     }
-    currentOp.setOpTraits(new OpTraits(null, -1, null));
+    currentOp.setOpTraits(new OpTraits(null, -1, null,
+        currentOp.getOpTraits().getNumReduceSinks()));
     for (Operator<? extends OperatorDesc> childOp : currentOp.getChildOperators()) {
       if ((childOp instanceof ReduceSinkOperator) || (childOp instanceof GroupByOperator)) {
         break;
@@ -351,7 +354,7 @@ public class ConvertJoinMapJoin implemen
 
     // we can set the traits for this join operator
     OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(),
-        tezBucketJoinProcCtx.getNumBuckets(), null);
+        tezBucketJoinProcCtx.getNumBuckets(), null, joinOp.getOpTraits().getNumReduceSinks());
     mapJoinOp.setOpTraits(opTraits);
     mapJoinOp.setStatistics(joinOp.getStatistics());
     setNumberOfBucketsOnChildren(mapJoinOp);
@@ -377,8 +380,7 @@ public class ConvertJoinMapJoin implemen
 
     ReduceSinkOperator bigTableRS =
         (ReduceSinkOperator) joinOp.getParentOperators().get(bigTablePosition);
-    int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits()
-            .getNumBuckets();
+    int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits().getNumBuckets();
 
     // the sort and bucket cols have to match on both sides for this
     // transformation of the join operation
@@ -425,13 +427,12 @@ public class ConvertJoinMapJoin implemen
   }
 
   /*
-   * If the parent reduce sink of the big table side has the same emit key cols
-   * as its parent, we can create a bucket map join eliminating the reduce sink.
+   * If the parent reduce sink of the big table side has the same emit key cols as its parent, we
+   * can create a bucket map join eliminating the reduce sink.
    */
   private boolean checkConvertJoinBucketMapJoin(JoinOperator joinOp,
       OptimizeTezProcContext context, int bigTablePosition,
-      TezBucketJoinProcCtx tezBucketJoinProcCtx)
-  throws SemanticException {
+      TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException {
     // bail on mux-operator because mux operator masks the emit keys of the
     // constituent reduce sinks
     if (!(joinOp.getParentOperators().get(0) instanceof ReduceSinkOperator)) {
@@ -453,8 +454,8 @@ public class ConvertJoinMapJoin implemen
     }
 
     /*
-     * this is the case when the big table is a sub-query and is probably
-     * already bucketed by the join column in say a group by operation
+     * this is the case when the big table is a sub-query and is probably already bucketed by the
+     * join column in say a group by operation
      */
     boolean isSubQuery = false;
     if (numBuckets < 0) {
@@ -484,10 +485,16 @@ public class ConvertJoinMapJoin implemen
         int colCount = 0;
         // parent op is guaranteed to have a single list because it is a reduce sink
         for (String colName : parentColNames.get(0)) {
+          if (listBucketCols.size() <= colCount) {
+            // can happen with virtual columns. RS would add the column to its output columns
+            // but it would not exist in the grandparent output columns or exprMap.
+            return false;
+          }
           // all columns need to be at least a subset of the parentOfParent's bucket cols
           ExprNodeDesc exprNodeDesc = colExprMap.get(colName);
           if (exprNodeDesc instanceof ExprNodeColumnDesc) {
-            if (((ExprNodeColumnDesc)exprNodeDesc).getColumn().equals(listBucketCols.get(colCount))) {
+            if (((ExprNodeColumnDesc) exprNodeDesc).getColumn()
+                .equals(listBucketCols.get(colCount))) {
               colCount++;
             } else {
               break;
@@ -557,14 +564,13 @@ public class ConvertJoinMapJoin implemen
 
       Statistics currInputStat = parentOp.getStatistics();
       if (currInputStat == null) {
-        LOG.warn("Couldn't get statistics from: "+parentOp);
+        LOG.warn("Couldn't get statistics from: " + parentOp);
         return -1;
       }
 
       long inputSize = currInputStat.getDataSize();
-      if ((bigInputStat == null) ||
-          ((bigInputStat != null) &&
-          (inputSize > bigInputStat.getDataSize()))) {
+      if ((bigInputStat == null)
+          || ((bigInputStat != null) && (inputSize > bigInputStat.getDataSize()))) {
 
         if (bigTableFound) {
           // cannot convert to map join; we've already chosen a big table
@@ -634,11 +640,11 @@ public class ConvertJoinMapJoin implemen
       }
     }
 
-    //can safely convert the join to a map join.
+    // can safely convert the join to a map join.
     MapJoinOperator mapJoinOp =
         MapJoinProcessor.convertJoinOpMapJoinOp(context.conf, joinOp,
-                joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(),
-                joinOp.getConf().getMapAliases(), bigTablePosition, true);
+            joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(),
+            joinOp.getConf().getMapAliases(), bigTablePosition, true);
 
     Operator<? extends OperatorDesc> parentBigTableOp =
         mapJoinOp.getParentOperators().get(bigTablePosition);
@@ -662,7 +668,7 @@ public class ConvertJoinMapJoin implemen
             parentBigTableOp.getParentOperators().get(0));
       }
       parentBigTableOp.getParentOperators().get(0).removeChild(parentBigTableOp);
-      for (Operator<? extends OperatorDesc> op : mapJoinOp.getParentOperators()) {
+      for (Operator<? extends OperatorDesc>op : mapJoinOp.getParentOperators()) {
         if (!(op.getChildOperators().contains(mapJoinOp))) {
           op.getChildOperators().add(mapJoinOp);
         }
@@ -676,7 +682,7 @@ public class ConvertJoinMapJoin implemen
   private boolean hasDynamicPartitionBroadcast(Operator<?> parent) {
     boolean hasDynamicPartitionPruning = false;
 
-    for (Operator<?> op: parent.getChildOperators()) {
+    for (Operator<?> op : parent.getChildOperators()) {
       while (op != null) {
         if (op instanceof AppMasterEventOperator && op.getConf() instanceof DynamicPruningEventDesc) {
           // found dynamic partition pruning operator

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java Tue Mar 10 17:49:39 2015
@@ -82,7 +82,7 @@ public class OpTraitsRulesProcFactory {
   }
 
   /*
-   * Reduce sink operator is the de-facto operator 
+   * Reduce sink operator is the de-facto operator
    * for determining keyCols (emit keys of a map phase)
    */
   public static class ReduceSinkRule implements NodeProcessor {
@@ -106,24 +106,25 @@ public class OpTraitsRulesProcFactory {
       List<List<String>> listBucketCols = new ArrayList<List<String>>();
       listBucketCols.add(bucketCols);
       int numBuckets = -1;
+      int numReduceSinks = 1;
       OpTraits parentOpTraits = rs.getParentOperators().get(0).getConf().getOpTraits();
       if (parentOpTraits != null) {
         numBuckets = parentOpTraits.getNumBuckets();
+        numReduceSinks += parentOpTraits.getNumReduceSinks();
       }
-      OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols);
+      OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols, numReduceSinks);
       rs.setOpTraits(opTraits);
       return null;
     }
   }
 
   /*
-   * Table scan has the table object and pruned partitions that has information such as
-   * bucketing, sorting, etc. that is used later for optimization.
+   * Table scan has the table object and pruned partitions that has information
+   * such as bucketing, sorting, etc. that is used later for optimization.
    */
   public static class TableScanRule implements NodeProcessor {
 
-    public boolean checkBucketedTable(Table tbl, 
-        ParseContext pGraphContext,
+    public boolean checkBucketedTable(Table tbl, ParseContext pGraphContext,
         PrunedPartitionList prunedParts) throws SemanticException {
 
       if (tbl.isPartitioned()) {
@@ -131,9 +132,11 @@ public class OpTraitsRulesProcFactory {
         // construct a mapping of (Partition->bucket file names) and (Partition -> bucket number)
         if (!partitions.isEmpty()) {
           for (Partition p : partitions) {
-            List<String> fileNames =
-                AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(), pGraphContext);
-            // The number of files for the table should be same as number of buckets.
+            List<String> fileNames = 
+                AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(), 
+                    pGraphContext);
+            // The number of files for the table should be same as number of
+            // buckets.
             int bucketCount = p.getBucketCount();
 
             if (fileNames.size() != 0 && fileNames.size() != bucketCount) {
@@ -143,8 +146,9 @@ public class OpTraitsRulesProcFactory {
         }
       } else {
 
-        List<String> fileNames =
-            AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(), pGraphContext);
+        List<String> fileNames = 
+            AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(), 
+                pGraphContext);
         Integer num = new Integer(tbl.getNumBuckets());
 
         // The number of files for the table should be same as number of buckets.
@@ -183,7 +187,8 @@ public class OpTraitsRulesProcFactory {
         }
         sortedColsList.add(sortCols);
       }
-      OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList);
+      // num reduce sinks hardcoded to 0 because TS has no parents
+      OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList, 0);
       ts.setOpTraits(opTraits);
       return null;
     }
@@ -208,8 +213,13 @@ public class OpTraitsRulesProcFactory {
       }
 
       List<List<String>> listBucketCols = new ArrayList<List<String>>();
+      int numReduceSinks = 0;
+      OpTraits parentOpTraits = gbyOp.getParentOperators().get(0).getOpTraits();
+      if (parentOpTraits != null) {
+        numReduceSinks = parentOpTraits.getNumReduceSinks();
+      }
       listBucketCols.add(gbyKeys);
-      OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols);
+      OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols, numReduceSinks);
       gbyOp.setOpTraits(opTraits);
       return null;
     }
@@ -217,8 +227,8 @@ public class OpTraitsRulesProcFactory {
 
   public static class SelectRule implements NodeProcessor {
 
-    public List<List<String>> getConvertedColNames(List<List<String>> parentColNames,
-        SelectOperator selOp) {
+    public List<List<String>> getConvertedColNames(
+        List<List<String>> parentColNames, SelectOperator selOp) {
       List<List<String>> listBucketCols = new ArrayList<List<String>>();
       if (selOp.getColumnExprMap() != null) {
         if (parentColNames != null) {
@@ -244,8 +254,8 @@ public class OpTraitsRulesProcFactory {
     @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
         Object... nodeOutputs) throws SemanticException {
-      SelectOperator selOp = (SelectOperator)nd;
-      List<List<String>> parentBucketColNames =
+      SelectOperator selOp = (SelectOperator) nd;
+      List<List<String>> parentBucketColNames = 
           selOp.getParentOperators().get(0).getOpTraits().getBucketColNames();
 
       List<List<String>> listBucketCols = null;
@@ -254,18 +264,21 @@ public class OpTraitsRulesProcFactory {
         if (parentBucketColNames != null) {
           listBucketCols = getConvertedColNames(parentBucketColNames, selOp);
         }
-        List<List<String>> parentSortColNames = selOp.getParentOperators().get(0).getOpTraits()
-            .getSortCols();
+        List<List<String>> parentSortColNames = 
+            selOp.getParentOperators().get(0).getOpTraits().getSortCols();
         if (parentSortColNames != null) {
           listSortCols = getConvertedColNames(parentSortColNames, selOp);
         }
       }
 
       int numBuckets = -1;
-      if (selOp.getParentOperators().get(0).getOpTraits() != null) {
-        numBuckets = selOp.getParentOperators().get(0).getOpTraits().getNumBuckets();
+      int numReduceSinks = 0;
+      OpTraits parentOpTraits = selOp.getParentOperators().get(0).getOpTraits();
+      if (parentOpTraits != null) {
+        numBuckets = parentOpTraits.getNumBuckets();
+        numReduceSinks = parentOpTraits.getNumReduceSinks();
       }
-      OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols);
+      OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols, numReduceSinks);
       selOp.setOpTraits(opTraits);
       return null;
     }
@@ -276,26 +289,31 @@ public class OpTraitsRulesProcFactory {
     @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
         Object... nodeOutputs) throws SemanticException {
-      JoinOperator joinOp = (JoinOperator)nd;
+      JoinOperator joinOp = (JoinOperator) nd;
       List<List<String>> bucketColsList = new ArrayList<List<String>>();
       List<List<String>> sortColsList = new ArrayList<List<String>>();
       byte pos = 0;
+      int numReduceSinks = 0; // will be set to the larger of the parents
       for (Operator<? extends OperatorDesc> parentOp : joinOp.getParentOperators()) {
         if (!(parentOp instanceof ReduceSinkOperator)) {
           // can be mux operator
           break;
         }
-        ReduceSinkOperator rsOp = (ReduceSinkOperator)parentOp;
+        ReduceSinkOperator rsOp = (ReduceSinkOperator) parentOp;
         if (rsOp.getOpTraits() == null) {
           ReduceSinkRule rsRule = new ReduceSinkRule();
           rsRule.process(rsOp, stack, procCtx, nodeOutputs);
         }
-        bucketColsList.add(getOutputColNames(joinOp, rsOp.getOpTraits().getBucketColNames(), pos));
-        sortColsList.add(getOutputColNames(joinOp, rsOp.getOpTraits().getSortCols(), pos));
+        OpTraits parentOpTraits = rsOp.getOpTraits();
+        bucketColsList.add(getOutputColNames(joinOp, parentOpTraits.getBucketColNames(), pos));
+        sortColsList.add(getOutputColNames(joinOp, parentOpTraits.getSortCols(), pos));
+        if (parentOpTraits.getNumReduceSinks() > numReduceSinks) {
+          numReduceSinks = parentOpTraits.getNumReduceSinks();
+        }
         pos++;
       }
 
-      joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList));
+      joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList, numReduceSinks));
       return null;
     }
 
@@ -311,7 +329,7 @@ public class OpTraitsRulesProcFactory {
         for (String colName : colNames) {
           for (ExprNodeDesc exprNode : joinOp.getConf().getExprs().get(pos)) {
             if (exprNode instanceof ExprNodeColumnDesc) {
-              if(((ExprNodeColumnDesc)(exprNode)).getColumn().equals(colName)) {
+              if (((ExprNodeColumnDesc) (exprNode)).getColumn().equals(colName)) {
                 for (Entry<String, ExprNodeDesc> entry : joinOp.getColumnExprMap().entrySet()) {
                   if (entry.getValue().isSame(exprNode)) {
                     bucketColNames.add(entry.getKey());
@@ -338,20 +356,30 @@ public class OpTraitsRulesProcFactory {
   }
 
   /*
-   *  When we have operators that have multiple parents, it is not
-   *  clear which parent's traits we need to propagate forward.
+   * When we have operators that have multiple parents, it is not clear which
+   * parent's traits we need to propagate forward.
    */
   public static class MultiParentRule implements NodeProcessor {
 
     @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
         Object... nodeOutputs) throws SemanticException {
-      OpTraits opTraits = new OpTraits(null, -1, null);
       @SuppressWarnings("unchecked")
-      Operator<? extends OperatorDesc> operator = (Operator<? extends OperatorDesc>)nd;
+      Operator<? extends OperatorDesc> operator = (Operator<? extends OperatorDesc>) nd;
+
+      int numReduceSinks = 0;
+      for (Operator<?> parentOp : operator.getParentOperators()) {
+        if (parentOp.getOpTraits() == null) {
+          continue;
+        }
+        if (parentOp.getOpTraits().getNumReduceSinks() > numReduceSinks) {
+          numReduceSinks = parentOp.getOpTraits().getNumReduceSinks();
+        }
+      }
+      OpTraits opTraits = new OpTraits(null, -1, null, numReduceSinks);
       operator.setOpTraits(opTraits);
       return null;
-    } 
+    }
   }
 
   public static NodeProcessor getTableScanRule() {
@@ -361,7 +389,7 @@ public class OpTraitsRulesProcFactory {
   public static NodeProcessor getReduceSinkRule() {
     return new ReduceSinkRule();
   }
-  
+
   public static NodeProcessor getSelectRule() {
     return new SelectRule();
   }

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java Tue Mar 10 17:49:39 2015
@@ -103,7 +103,7 @@ public class SparkMapJoinOptimizer imple
     }
 
     // we can set the traits for this join operator
-    OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null);
+    OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null, joinOp.getOpTraits().getNumReduceSinks());
     mapJoinOp.setOpTraits(opTraits);
     mapJoinOp.setStatistics(joinOp.getStatistics());
     setNumberOfBucketsOnChildren(mapJoinOp);

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java Tue Mar 10 17:49:39 2015
@@ -718,7 +718,7 @@ public class CalcitePlanner extends Sema
       hepPgmBldr.addRuleInstance(ReduceExpressionsRule.JOIN_INSTANCE);
       hepPgmBldr.addRuleInstance(ReduceExpressionsRule.FILTER_INSTANCE);
       hepPgmBldr.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE);
-      hepPgmBldr.addRuleInstance(ProjectRemoveRule.NAME_CALC_INSTANCE);
+      hepPgmBldr.addRuleInstance(ProjectRemoveRule.INSTANCE);
       hepPgmBldr.addRuleInstance(UnionMergeRule.INSTANCE);
 
       hepPgm = hepPgmBldr.build();
@@ -792,7 +792,7 @@ public class CalcitePlanner extends Sema
       RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY,
           HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY,
           RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
-          HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY, true);
+          HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY);
       basePlan = fieldTrimmer.trim(basePlan);
 
       // 5. Rerun PPD through Project as column pruning would have introduced DT

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Mar 10 17:49:39 2015
@@ -555,7 +555,7 @@ public class TypeCheckProcFactory {
 
   private static ExprNodeDesc toExprNodeDesc(ColumnInfo colInfo) {
     ObjectInspector inspector = colInfo.getObjectInspector();
-    if (inspector instanceof ConstantObjectInspector && 
+    if (inspector instanceof ConstantObjectInspector &&
         inspector instanceof PrimitiveObjectInspector) {
       PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
       Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
@@ -994,8 +994,15 @@ public class TypeCheckProcFactory {
           .getText());
       // NOTE: tableAlias must be a valid non-ambiguous table alias,
       // because we've checked that in TOK_TABLE_OR_COL's process method.
-      ColumnInfo colInfo = input.get(tableAlias, ((ExprNodeConstantDesc) nodeOutputs[1]).getValue()
-          .toString());
+      String colName;
+      if (nodeOutputs[1] instanceof ExprNodeConstantDesc) {
+        colName = ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString();
+      }  else if (nodeOutputs[1] instanceof ExprNodeColumnDesc) {
+        colName = ((ExprNodeColumnDesc)nodeOutputs[1]).getColumn();
+      } else {
+        throw new SemanticException("Unexpected ExprNode : " + nodeOutputs[1]);
+      }
+      ColumnInfo colInfo = input.get(tableAlias, colName);
 
       if (colInfo == null) {
         ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);

Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java?rev=1665637&r1=1665636&r2=1665637&view=diff
==============================================================================
--- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java (original)
+++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java Tue Mar 10 17:49:39 2015
@@ -25,11 +25,14 @@ public class OpTraits {
   List<List<String>> bucketColNames;
   List<List<String>> sortColNames;
   int numBuckets;
+  int numReduceSinks;
 
-  public OpTraits(List<List<String>> bucketColNames, int numBuckets, List<List<String>> sortColNames) {
+  public OpTraits(List<List<String>> bucketColNames, int numBuckets,
+      List<List<String>> sortColNames, int numReduceSinks) {
     this.bucketColNames = bucketColNames;
     this.numBuckets = numBuckets;
     this.sortColNames = sortColNames;
+    this.numReduceSinks = numReduceSinks;
   }
 
   public List<List<String>> getBucketColNames() {
@@ -55,4 +58,12 @@ public class OpTraits {
   public List<List<String>> getSortCols() {
     return sortColNames;
   }
+
+  public void setNumReduceSinks(int numReduceSinks) {
+    this.numReduceSinks = numReduceSinks;
+  }
+
+  public int getNumReduceSinks() {
+    return this.numReduceSinks;
+  }
 }



Mime
View raw message