Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 014E117BDC for ; Tue, 10 Mar 2015 17:49:49 +0000 (UTC) Received: (qmail 92149 invoked by uid 500); 10 Mar 2015 17:49:42 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 92099 invoked by uid 500); 10 Mar 2015 17:49:42 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 92088 invoked by uid 99); 10 Mar 2015 17:49:42 -0000 Received: from eris.apache.org (HELO hades.apache.org) (140.211.11.105) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Mar 2015 17:49:42 +0000 Received: from hades.apache.org (localhost [127.0.0.1]) by hades.apache.org (ASF Mail Server at hades.apache.org) with ESMTP id 73C19AC02E7 for ; Tue, 10 Mar 2015 17:49:42 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1665637 [1/2] - in /hive/branches/parquet: ./ beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/ beeline/src/test/org/apache/hive/beeline/ beeline/src/test/resources/ common/src/java/org/apache/hadoop/hive/conf/ dev-supp... Date: Tue, 10 Mar 2015 17:49:41 -0000 To: commits@hive.apache.org From: sunchao@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20150310174942.73C19AC02E7@hades.apache.org> Author: sunchao Date: Tue Mar 10 17:49:39 2015 New Revision: 1665637 URL: http://svn.apache.org/r1665637 Log: Merge trunk into parquet branch Added: hive/branches/parquet/beeline/src/test/resources/ - copied from r1665631, hive/trunk/beeline/src/test/resources/ hive/branches/parquet/dev-support/jenkins-execute-hms-test.sh - copied unchanged from r1665631, hive/trunk/dev-support/jenkins-execute-hms-test.sh hive/branches/parquet/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.12.0-to-0.13.0.mssql.sql - copied unchanged from r1665631, hive/trunk/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.12.0-to-0.13.0.mssql.sql hive/branches/parquet/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.13.0-to-0.14.0.mssql.sql - copied unchanged from r1665631, hive/trunk/metastore/scripts/upgrade/mssql/pre-0-upgrade-0.13.0-to-0.14.0.mssql.sql hive/branches/parquet/ql/src/test/queries/clientpositive/tez_join.q - copied unchanged from r1665631, hive/trunk/ql/src/test/queries/clientpositive/tez_join.q hive/branches/parquet/ql/src/test/results/clientpositive/tez/tez_join.q.out - copied unchanged from r1665631, hive/trunk/ql/src/test/results/clientpositive/tez/tez_join.q.out hive/branches/parquet/testutils/metastore/ - copied from r1665631, hive/trunk/testutils/metastore/ Modified: hive/branches/parquet/ (props changed) hive/branches/parquet/LICENSE hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java hive/branches/parquet/beeline/src/main/resources/BeeLine.properties hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java hive/branches/parquet/itests/src/test/resources/testconfiguration.properties hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java hive/branches/parquet/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q hive/branches/parquet/ql/src/test/results/clientpositive/auto_join11.q.out hive/branches/parquet/ql/src/test/results/clientpositive/auto_join12.q.out hive/branches/parquet/ql/src/test/results/clientpositive/auto_join13.q.out hive/branches/parquet/ql/src/test/results/clientpositive/auto_join27.q.out hive/branches/parquet/ql/src/test/results/clientpositive/auto_join4.q.out hive/branches/parquet/ql/src/test/results/clientpositive/auto_join5.q.out hive/branches/parquet/ql/src/test/results/clientpositive/auto_join8.q.out hive/branches/parquet/ql/src/test/results/clientpositive/constantPropagateForSubQuery.q.out hive/branches/parquet/ql/src/test/results/clientpositive/correlationoptimizer1.q.out hive/branches/parquet/ql/src/test/results/clientpositive/correlationoptimizer6.q.out hive/branches/parquet/ql/src/test/results/clientpositive/explain_logical.q.out hive/branches/parquet/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out hive/branches/parquet/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out hive/branches/parquet/ql/src/test/results/clientpositive/metadataonly1.q.out hive/branches/parquet/ql/src/test/results/clientpositive/ppd_join_filter.q.out hive/branches/parquet/ql/src/test/results/clientpositive/ppd_vc.q.out hive/branches/parquet/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_1.q.out hive/branches/parquet/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out hive/branches/parquet/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out hive/branches/parquet/ql/src/test/results/clientpositive/subquery_in_having.q.out hive/branches/parquet/ql/src/test/results/clientpositive/subquery_notin.q.out hive/branches/parquet/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out hive/branches/parquet/ql/src/test/results/clientpositive/table_access_keys_stats.q.out hive/branches/parquet/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out hive/branches/parquet/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out hive/branches/parquet/ql/src/test/results/clientpositive/tez/mrr.q.out hive/branches/parquet/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out hive/branches/parquet/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java hive/branches/parquet/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java hive/branches/parquet/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java Propchange: hive/branches/parquet/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Tue Mar 10 17:49:39 2015 @@ -4,4 +4,4 @@ /hive/branches/spark:1608589-1660298 /hive/branches/tez:1494760-1622766 /hive/branches/vectorization:1466908-1527856 -/hive/trunk:1656617-1664450 +/hive/trunk:1656617-1665631 Modified: hive/branches/parquet/LICENSE URL: http://svn.apache.org/viewvc/hive/branches/parquet/LICENSE?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/LICENSE (original) +++ hive/branches/parquet/LICENSE Tue Mar 10 17:49:39 2015 @@ -465,4 +465,33 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTI AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +OF THE POSSIBILITY OF SUCH DAMAGE. + +For the PostgreSQL JDBC driver jar file: + +Copyright (c) 1997-2011, PostgreSQL Global Development Group +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of the PostgreSQL Global Development Group nor the names + of its contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file Modified: hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java (original) +++ hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/BeeLine.java Tue Mar 10 17:49:39 2015 @@ -28,11 +28,9 @@ import java.io.Closeable; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; -import java.io.PrintWriter; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -53,6 +51,7 @@ import java.sql.Statement; import java.text.ChoiceFormat; import java.text.MessageFormat; import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -82,7 +81,6 @@ import jline.console.ConsoleReader; import jline.console.history.History; import jline.console.history.FileHistory; -import jline.internal.Log; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.OptionBuilder; @@ -157,6 +155,8 @@ public class BeeLine implements Closeabl "xmlelements", new XMLElementOutputFormat(this), }); + private List supportedLocalDriver = + new ArrayList(Arrays.asList("com.mysql.jdbc.Driver", "org.postgresql.Driver")); final CommandHandler[] commandHandlers = new CommandHandler[] { new ReflectiveCommandHandler(this, new String[] {"quit", "done", "exit"}, @@ -249,6 +249,10 @@ public class BeeLine implements Closeabl null), new ReflectiveCommandHandler(this, new String[] {"nullemptystring"}, new Completer[] {new BooleanCompleter()}), + new ReflectiveCommandHandler(this, new String[]{"addlocaldriverjar"}, + null), + new ReflectiveCommandHandler(this, new String[]{"addlocaldrivername"}, + null) }; @@ -1579,6 +1583,11 @@ public class BeeLine implements Closeabl return true; } + // find whether exists a local driver to accept the url + if (findLocalDriver(url) != null) { + return true; + } + return false; } catch (Exception e) { debug(e.toString()); @@ -1601,6 +1610,40 @@ public class BeeLine implements Closeabl return null; } + public Driver findLocalDriver(String url) throws Exception { + if(drivers == null){ + return null; + } + + for (Driver d : drivers) { + try { + String clazzName = d.getClass().getName(); + Driver driver = (Driver) Class.forName(clazzName, true, + Thread.currentThread().getContextClassLoader()).newInstance(); + if (driver.acceptsURL(url) && isSupportedLocalDriver(driver)) { + return driver; + } + } catch (SQLException e) { + error(e); + throw new Exception(e); + } + } + return null; + } + + public boolean isSupportedLocalDriver(Driver driver) { + String driverName = driver.getClass().getName(); + for (String name : supportedLocalDriver) { + if (name.equals(driverName)) { + return true; + } + } + return false; + } + + public void addLocalDriverClazz(String driverClazz) { + supportedLocalDriver.add(driverClazz); + } Driver[] scanDrivers(String line) throws IOException { return scanDrivers(false); Modified: hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java (original) +++ hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/ClassNameCompleter.java Tue Mar 10 17:49:39 2015 @@ -52,9 +52,12 @@ package org.apache.hive.beeline; import jline.console.completer.StringsCompleter; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import java.io.File; import java.io.IOException; +import java.io.FileInputStream; import java.net.JarURLConnection; import java.net.URL; import java.net.URLClassLoader; @@ -62,11 +65,15 @@ import java.net.URLConnection; import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; +import java.util.List; +import java.util.ArrayList; import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.Enumeration; import java.util.TreeSet; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipEntry; /** * the completer is original provided in JLine 0.9.94 and is being removed in 2.12. Add the @@ -74,6 +81,10 @@ import java.util.TreeSet; */ public class ClassNameCompleter extends StringsCompleter { + private static final Log LOG = LogFactory.getLog(ClassNameCompleter.class.getName()); + public final static String clazzFileNameExtension = ".class"; + public final static String jarFileNameExtension = ".jar"; + public ClassNameCompleter(String... candidates) { super(candidates); } @@ -81,7 +92,7 @@ public class ClassNameCompleter extends public static String[] getClassNames() throws IOException { Set urls = new HashSet(); - for (ClassLoader loader = ClassNameCompleter.class.getClassLoader(); loader != null; + for (ClassLoader loader = Thread.currentThread().getContextClassLoader(); loader != null; loader = loader.getParent()) { if (!(loader instanceof URLClassLoader)) { continue; @@ -97,7 +108,7 @@ public class ClassNameCompleter extends for (int i = 0; i < systemClasses.length; i++) { URL classURL = systemClasses[i] - .getResource("/" + systemClasses[i].getName().replace('.', '/') + ".class"); + .getResource("/" + systemClasses[i].getName().replace('.', '/') + clazzFileNameExtension); if (classURL != null) { URLConnection uc = classURL.openConnection(); @@ -136,12 +147,14 @@ public class ClassNameCompleter extends String name = entry.getName(); - if (!name.endsWith(".class")) { + if (isClazzFile(name)) { /* only use class file*/ + classes.add(name); + } else if (isJarFile(name)) { + classes.addAll(getClassNamesFromJar(name)); + } else { continue; } - - classes.add(name); } } @@ -151,8 +164,7 @@ public class ClassNameCompleter extends for (Iterator i = classes.iterator(); i.hasNext(); ) { String name = (String) i.next(); - classNames.add(name.replace('/', '.'). - substring(0, name.length() - 6)); + classNames.add(name.replace('/', '.').substring(0, name.length() - 6)); } return (String[]) classNames.toArray(new String[classNames.size()]); @@ -173,7 +185,7 @@ public class ClassNameCompleter extends continue; } else if (files[i].isDirectory()) { getClassFiles(root, holder, files[i], maxDirectories); - } else if (files[i].getName().endsWith(".class")) { + } else if (files[i].getName().endsWith(clazzFileNameExtension)) { holder.add(files[i].getAbsolutePath(). substring(root.length() + 1)); } @@ -181,4 +193,54 @@ public class ClassNameCompleter extends return holder; } + + /** + * Get clazz names from a jar file path + * @param path specifies the jar file's path + * @return + */ + private static List getClassNamesFromJar(String path) { + List classNames = new ArrayList(); + ZipInputStream zip = null; + try { + zip = new ZipInputStream(new FileInputStream(path)); + ZipEntry entry = zip.getNextEntry(); + while (entry != null) { + if (!entry.isDirectory() && entry.getName().endsWith(clazzFileNameExtension)) { + StringBuilder className = new StringBuilder(); + for (String part : entry.getName().split("/")) { + if (className.length() != 0) { + className.append("."); + } + className.append(part); + if (part.endsWith(clazzFileNameExtension)) { + className.setLength(className.length() - clazzFileNameExtension.length()); + } + } + classNames.add(className.toString()); + } + entry = zip.getNextEntry(); + } + } catch (IOException e) { + LOG.error("Fail to parse the class name from the Jar file due to the exception:" + e); + } finally { + if (zip != null) { + try { + zip.close(); + } catch (IOException e) { + LOG.error("Fail to close the file due to the exception:" + e); + } + } + } + + return classNames; + } + + private static boolean isJarFile(String fileName) { + return fileName.endsWith(jarFileNameExtension); + } + + private static boolean isClazzFile(String clazzName) { + return clazzName.endsWith(clazzFileNameExtension); + } } Modified: hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java (original) +++ hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/Commands.java Tue Mar 10 17:49:39 2015 @@ -25,12 +25,16 @@ package org.apache.hive.beeline; import org.apache.hadoop.io.IOUtils; import java.io.BufferedReader; +import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -126,6 +130,46 @@ public class Commands { return true; } + public boolean addlocaldrivername(String line) { + String driverName = arg1(line, "driver class name"); + try { + beeLine.setDrivers(Arrays.asList(beeLine.scanDrivers(false))); + } catch (IOException e) { + beeLine.error("Fail to scan drivers due to the exception:" + e); + beeLine.error(e); + } + for (Driver d : beeLine.getDrivers()) { + if (driverName.equals(d.getClass().getName())) { + beeLine.addLocalDriverClazz(driverName); + return true; + } + } + beeLine.error("Fail to find a driver which contains the driver class"); + return false; + } + + public boolean addlocaldriverjar(String line) { + // If jar file is in the hdfs, it should be downloaded first. + String jarPath = arg1(line, "jar path"); + File p = new File(jarPath); + if (!p.exists()) { + beeLine.error("The jar file in the path " + jarPath + " can't be found!"); + return false; + } + + URLClassLoader classLoader = (URLClassLoader) Thread.currentThread().getContextClassLoader(); + try { + beeLine.debug(jarPath + " is added to the local beeline."); + URLClassLoader newClassLoader = new URLClassLoader(new URL[]{p.toURL()}, classLoader); + + Thread.currentThread().setContextClassLoader(newClassLoader); + beeLine.setDrivers(Arrays.asList(beeLine.scanDrivers(false))); + } catch (Exception e) { + beeLine.error("Fail to add local jar due to the exception:" + e); + beeLine.error(e); + } + return true; + } public boolean history(String line) { Iterator hist = beeLine.getConsoleReader().getHistory().entries(); Modified: hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java (original) +++ hive/branches/parquet/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java Tue Mar 10 17:49:39 2015 @@ -25,9 +25,12 @@ package org.apache.hive.beeline; import java.io.IOException; import java.sql.Connection; import java.sql.DatabaseMetaData; +import java.sql.Driver; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -112,9 +115,9 @@ class DatabaseConnection { return beeLine.error(cnfe); } - boolean foundDriver = false; + boolean isDriverRegistered = false; try { - foundDriver = DriverManager.getDriver(getUrl()) != null; + isDriverRegistered = DriverManager.getDriver(getUrl()) != null; } catch (Exception e) { } @@ -134,7 +137,13 @@ class DatabaseConnection { info.put(HIVE_CONF_PREFIX + var.getKey(), var.getValue()); } - setConnection(DriverManager.getConnection(getUrl(), info)); + if (isDriverRegistered) { + // if the driver registered in the driver manager, get the connection via the driver manager + setConnection(DriverManager.getConnection(getUrl(), info)); + } else { + beeLine.debug("Use the driver from local added jar file."); + setConnection(getConnectionFromLocalDriver(getUrl(), info)); + } setDatabaseMetaData(getConnection().getMetaData()); try { @@ -170,6 +179,26 @@ class DatabaseConnection { return true; } + public Connection getConnectionFromLocalDriver(String url, Properties properties) { + Collection drivers = beeLine.getDrivers(); + for (Driver d : drivers) { + try { + if (d.acceptsURL(url) && beeLine.isSupportedLocalDriver(d)) { + String clazzName = d.getClass().getName(); + beeLine.debug("Driver name is " + clazzName); + Driver driver = + (Driver) Class.forName(clazzName, true, Thread.currentThread().getContextClassLoader()) + .newInstance(); + return driver.connect(url, properties); + } + } catch (Exception e) { + beeLine.error("Fail to connect with a local driver due to the exception:" + e); + beeLine.error(e); + } + } + return null; + } + public Connection getConnection() throws SQLException { if (connection != null) { Modified: hive/branches/parquet/beeline/src/main/resources/BeeLine.properties URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/main/resources/BeeLine.properties?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/beeline/src/main/resources/BeeLine.properties (original) +++ hive/branches/parquet/beeline/src/main/resources/BeeLine.properties Tue Mar 10 17:49:39 2015 @@ -71,6 +71,8 @@ help-properties: Connect to the database help-outputformat: Set the output format for displaying results (table,vertical,csv2,dsv,tsv2,xmlattrs,xmlelements, and deprecated formats(csv, tsv)) help-delimiterForDSV: Set the delimiter for dsv output format help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false. +help-addlocaldriverjar: Add driver jar file in the beeline client side. +help-addlocaldrivername: Add driver name that needs to be supported in the beeline client side. jline-missing: The JLine jar was not found. Please ensure it is installed. @@ -174,6 +176,8 @@ cmd-usage: Usage: java org.apache.hive.c \ --delimiterForDSV=DELIMITER specify the delimiter for delimiter-separated values output format (default: |)\n \ \ --isolation=LEVEL set the transaction isolation level\n \ \ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \ +\ --addlocaldriverjar=DRIVERJARNAME Add driver jar file in the beeline client side\n \ +\ --addlocaldrivername=DRIVERNAME Add drvier name needs to be supported in the beeline client side\n \ \ --help display this message Modified: hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java (original) +++ hive/branches/parquet/beeline/src/test/org/apache/hive/beeline/TestBeelineArgParsing.java Tue Mar 10 17:49:39 2015 @@ -19,19 +19,40 @@ package org.apache.hive.beeline; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.io.File; import java.io.FileOutputStream; -import junit.framework.Assert; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hive.common.util.HiveTestUtils; +import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; /** * Unit test for Beeline arg parser. */ +@RunWith(Parameterized.class) public class TestBeelineArgParsing { + private static final Log LOG = LogFactory.getLog(TestBeelineArgParsing.class.getName()); + private String connectionString; + private String driverClazzName; + private String driverJarFileName; + private boolean defaultSupported; + + public TestBeelineArgParsing(String connectionString, String driverClazzName, String driverJarFileName, + boolean defaultSupported) { + this.connectionString = connectionString; + this.driverClazzName = driverClazzName; + this.driverJarFileName = driverJarFileName; + this.defaultSupported = defaultSupported; + } public class TestBeeline extends BeeLine { @@ -52,6 +73,27 @@ public class TestBeelineArgParsing { } return true; } + + public boolean addlocaldrivername(String driverName) { + String line = "addlocaldrivername " + driverName; + return getCommands().addlocaldrivername(line); + } + + public boolean addLocalJar(String url){ + String line = "addlocaldriverjar " + url; + return getCommands().addlocaldriverjar(line); + } + } + + @Parameters + public static Collection data() { + return Arrays.asList( + new Object[][] { + {"jdbc:postgresql://host:5432/testdb", "org.postgresql.Driver", "postgresql-9.3.jdbc3.jar", + true}, + {"jdbc:dummy://host:5432/testdb", "org.apache.dummy.DummyDriver", + "DummyDriver-1.0-SNAPSHOT.jar", false} + }); } @Test @@ -59,7 +101,7 @@ public class TestBeelineArgParsing { TestBeeline bl = new TestBeeline(); String args[] = new String[] {"-u", "url", "-n", "name", "-p", "password", "-d", "driver", "-a", "authType"}; - Assert.assertEquals(0, bl.initArgs(args)); + org.junit.Assert.assertEquals(0, bl.initArgs(args)); Assert.assertTrue(bl.connectArgs.equals("url name password driver")); Assert.assertTrue(bl.getOpts().getAuthType().equals("authType")); } @@ -169,4 +211,32 @@ public class TestBeelineArgParsing { Assert.assertEquals(-1, bl.initArgs(args)); } + @Test + public void testAddLocalJar() throws Exception { + TestBeeline bl = new TestBeeline(); + Assert.assertNull(bl.findLocalDriver(connectionString)); + + LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName); + String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName); + + bl.addLocalJar(mysqlDriverPath); + bl.addlocaldrivername(driverClazzName); + Assert.assertEquals(bl.findLocalDriver(connectionString).getClass().getName(), driverClazzName); + } + + @Test + public void testAddLocalJarWithoutAddDriverClazz() throws Exception { + TestBeeline bl = new TestBeeline(); + + LOG.info("Add " + driverJarFileName + " for the driver class " + driverClazzName); + String mysqlDriverPath = HiveTestUtils.getFileFromClasspath(driverJarFileName); + + bl.addLocalJar(mysqlDriverPath); + if (!defaultSupported) { + Assert.assertNull(bl.findLocalDriver(connectionString)); + } else { + // no need to add for the default supported local jar driver + Assert.assertEquals(bl.findLocalDriver(connectionString).getClass().getName(), driverClazzName); + } + } } Modified: hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original) +++ hive/branches/parquet/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Mar 10 17:49:39 2015 @@ -601,6 +601,9 @@ public class HiveConf extends Configurat + "is set to instance of HiveAuthorizerFactory, then this value is ignored."), FIRE_EVENTS_FOR_DML("hive.metastore.dml.events", false, "If true, the metastore will be asked" + " to fire events for DML operations"), + METASTORE_CLIENT_DROP_PARTITIONS_WITH_EXPRESSIONS("hive.metastore.client.drop.partitions.using.expressions", true, + "Choose whether dropping partitions with HCatClient pushes the partition-predicate to the metastore, " + + "or drops partitions iteratively"), // Parameters for exporting metadata on table drop (requires the use of the) // org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener Modified: hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml (original) +++ hive/branches/parquet/hcatalog/webhcat/java-client/pom.xml Tue Mar 10 17:49:39 2015 @@ -45,6 +45,11 @@ hive-hcatalog-core ${project.version} + + org.apache.hive + hive-exec + ${project.version} + org.apache.hive Modified: hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java (original) +++ hive/branches/parquet/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java Tue Mar 10 17:49:39 2015 @@ -20,12 +20,15 @@ package org.apache.hive.hcatalog.api; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; -import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.conf.HiveConf; @@ -36,6 +39,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -48,12 +52,25 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; +import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils; import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The HCatClientHMSImpl is the Hive Metastore client based implementation of @@ -61,6 +78,7 @@ import org.apache.thrift.TException; */ public class HCatClientHMSImpl extends HCatClient { + private static final Logger LOG = LoggerFactory.getLogger(HCatClientHMSImpl.class); private HiveMetaStoreClient hmsClient; private Configuration config; private HiveConf hiveConfig; @@ -480,19 +498,122 @@ public class HCatClientHMSImpl extends H } } + /** + * Helper class to help build ExprDesc tree to represent the partitions to be dropped. + * Note: At present, the ExpressionBuilder only constructs partition predicates where + * partition-keys equal specific values, and logical-AND expressions. E.g. + * ( dt = '20150310' AND region = 'US' ) + * This only supports the partition-specs specified by the Map argument of: + * {@link org.apache.hive.hcatalog.api.HCatClient#dropPartitions(String, String, Map, boolean)} + */ + private static class ExpressionBuilder { + + private Map partColumnTypesMap = Maps.newHashMap(); + private Map partSpecs; + + public ExpressionBuilder(Table table, Map partSpecs) { + this.partSpecs = partSpecs; + for (FieldSchema partField : table.getPartitionKeys()) { + partColumnTypesMap.put(partField.getName().toLowerCase(), + TypeInfoFactory.getPrimitiveTypeInfo(partField.getType())); + } + } + + private PrimitiveTypeInfo getTypeFor(String partColumn) { + return partColumnTypesMap.get(partColumn.toLowerCase()); + } + + private Object getTypeAppropriateValueFor(PrimitiveTypeInfo type, String value) { + ObjectInspectorConverters.Converter converter = ObjectInspectorConverters.getConverter( + TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(TypeInfoFactory.stringTypeInfo), + TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(type)); + + return converter.convert(value); + } + + public ExprNodeGenericFuncDesc equalityPredicate(String partColumn, String value) throws SemanticException { + + PrimitiveTypeInfo partColumnType = getTypeFor(partColumn); + ExprNodeColumnDesc partColumnExpr = new ExprNodeColumnDesc(partColumnType, partColumn, null, true); + ExprNodeConstantDesc valueExpr = new ExprNodeConstantDesc(partColumnType, + getTypeAppropriateValueFor(partColumnType, value)); + + return binaryPredicate("=", partColumnExpr, valueExpr); + } + + public ExprNodeGenericFuncDesc binaryPredicate(String function, ExprNodeDesc lhs, ExprNodeDesc rhs) throws SemanticException { + return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, + FunctionRegistry.getFunctionInfo(function).getGenericUDF(), + Lists.newArrayList(lhs, rhs)); + } + + public ExprNodeGenericFuncDesc build() throws SemanticException { + ExprNodeGenericFuncDesc resultExpr = null; + + for (Map.Entry partSpec : partSpecs.entrySet()) { + String column = partSpec.getKey(); + String value = partSpec.getValue(); + ExprNodeGenericFuncDesc partExpr = equalityPredicate(column, value); + + resultExpr = (resultExpr == null? partExpr : binaryPredicate("and", resultExpr, partExpr)); + } + + return resultExpr; + } + } // class ExpressionBuilder; + + private static boolean isExternal(Table table) { + return table.getParameters() != null + && "TRUE".equalsIgnoreCase(table.getParameters().get("EXTERNAL")); + } + + private void dropPartitionsUsingExpressions(Table table, Map partitionSpec, boolean ifExists) + throws SemanticException, TException { + LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions."); + ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build(); + ObjectPair serializedPartitionExpression = + new ObjectPair(partitionSpec.size(), + Utilities.serializeExpressionToKryo(partitionExpression)); + hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression), + !isExternal(table), // Delete data? + false, // Ignore Protection? + ifExists, // Fail if table doesn't exist? + false); // Need results back? + } + + private void dropPartitionsIteratively(String dbName, String tableName, + Map partitionSpec, boolean ifExists) throws HCatException, TException { + LOG.info("HCatClient: Dropping partitions iteratively."); + List partitions = hmsClient.listPartitionsByFilter(dbName, tableName, + getFilterString(partitionSpec), (short) -1); + for (Partition partition : partitions) { + dropPartition(partition, ifExists); + } + } + @Override public void dropPartitions(String dbName, String tableName, Map partitionSpec, boolean ifExists) throws HCatException { + LOG.info("HCatClient dropPartitions(db=" + dbName + ",table=" + tableName + ", partitionSpec: ["+ partitionSpec + "])."); try { dbName = checkDB(dbName); - List partitions = hmsClient.listPartitionsByFilter(dbName, tableName, - getFilterString(partitionSpec), (short)-1); + Table table = hmsClient.getTable(dbName, tableName); - for (Partition partition : partitions) { - dropPartition(partition, ifExists); + if (hiveConfig.getBoolVar(HiveConf.ConfVars.METASTORE_CLIENT_DROP_PARTITIONS_WITH_EXPRESSIONS)) { + try { + dropPartitionsUsingExpressions(table, partitionSpec, ifExists); + } + catch (SemanticException parseFailure) { + LOG.warn("Could not push down partition-specification to back-end, for dropPartitions(). Resorting to iteration.", + parseFailure); + dropPartitionsIteratively(dbName, tableName, partitionSpec, ifExists); + } + } + else { + // Not using expressions. + dropPartitionsIteratively(dbName, tableName, partitionSpec, ifExists); } - } catch (NoSuchObjectException e) { throw new ObjectNotFoundException( "NoSuchObjectException while dropping partition. " + Modified: hive/branches/parquet/itests/src/test/resources/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/branches/parquet/itests/src/test/resources/testconfiguration.properties?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/itests/src/test/resources/testconfiguration.properties (original) +++ hive/branches/parquet/itests/src/test/resources/testconfiguration.properties Tue Mar 10 17:49:39 2015 @@ -305,7 +305,8 @@ minitez.query.files=bucket_map_join_tez1 tez_smb_main.q,\ tez_smb_1.q,\ vectorized_dynamic_partition_pruning.q,\ - tez_multi_union.q + tez_multi_union.q,\ + tez_join.q encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_insert_partition_static.q,\ Modified: hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql (original) +++ hive/branches/parquet/metastore/scripts/upgrade/mssql/002-HIVE-7784.mssql.sql Tue Mar 10 17:49:39 2015 @@ -1,32 +1 @@ --- --- Create the table if it doesn't exist. --- -if not exists (SELECT 1 FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_NAME='PART_COL_STATS') -CREATE TABLE PART_COL_STATS -( - CS_ID bigint NOT NULL, - AVG_COL_LEN float NULL, - "COLUMN_NAME" varchar(128) NOT NULL, - COLUMN_TYPE varchar(128) NOT NULL, - DB_NAME varchar(128) NOT NULL, - BIG_DECIMAL_HIGH_VALUE varchar(255) NULL, - BIG_DECIMAL_LOW_VALUE varchar(255) NULL, - DOUBLE_HIGH_VALUE float NULL, - DOUBLE_LOW_VALUE float NULL, - LAST_ANALYZED bigint NOT NULL, - LONG_HIGH_VALUE bigint NULL, - LONG_LOW_VALUE bigint NULL, - MAX_COL_LEN bigint NULL, - NUM_DISTINCTS bigint NULL, - NUM_FALSES bigint NULL, - NUM_NULLS bigint NOT NULL, - NUM_TRUES bigint NULL, - PART_ID bigint NULL, - PARTITION_NAME varchar(767) NOT NULL, - "TABLE_NAME" varchar(128) NOT NULL -) -go - - CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (DB_NAME,TABLE_NAME,COLUMN_NAME,PARTITION_NAME); Modified: hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql (original) +++ hive/branches/parquet/metastore/scripts/upgrade/mssql/004-HIVE-8550.mssql.sql Tue Mar 10 17:49:39 2015 @@ -2,12 +2,9 @@ DROP INDEX PARTITIONS.UNIQUEPARTITION; ALTER TABLE PARTITIONS ALTER COLUMN PART_NAME NVARCHAR(767) NULL; CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (PART_NAME,TBL_ID); -GO --ALTER SDS.LOCATION from varchar to nvarchar ALTER TABLE SDS ALTER COLUMN LOCATION NVARCHAR(4000) NULL; -GO --ALTER PARTITION_KEY_VALS.PART_KEY_VAL from varchar to nvarchar ALTER TABLE PARTITION_KEY_VALS ALTER COLUMN PART_KEY_VAL NVARCHAR(255) NULL; -GO \ No newline at end of file Modified: hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql (original) +++ hive/branches/parquet/metastore/scripts/upgrade/mssql/005-HIVE-9296.mssql.sql Tue Mar 10 17:49:39 2015 @@ -8,19 +8,18 @@ CREATE TABLE NOTIFICATION_LOG TBL_NAME varchar(128) NULL, MESSAGE text NULL ); -GO + ALTER TABLE NOTIFICATION_LOG ADD CONSTRAINT NOTIFICATION_LOG_PK PRIMARY KEY (NL_ID); -GO CREATE TABLE NOTIFICATION_SEQUENCE ( NNI_ID bigint NOT NULL, NEXT_EVENT_ID bigint NOT NULL ); -GO + ALTER TABLE NOTIFICATION_SEQUENCE ADD CONSTRAINT NOTIFICATION_SEQUENCE_PK PRIMARY KEY (NNI_ID); -GO + Modified: hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original) +++ hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Tue Mar 10 17:49:39 2015 @@ -4142,12 +4142,16 @@ public class HiveMetaStore extends Thrif @Override public TableStatsResult get_table_statistics_req(TableStatsRequest request) throws MetaException, NoSuchObjectException, TException { - String dbName = request.getDbName(), tblName = request.getTblName(); + String dbName = request.getDbName().toLowerCase(); + String tblName = request.getTblName().toLowerCase(); startFunction("get_table_statistics_req: db=" + dbName + " table=" + tblName); TableStatsResult result = null; + List lowerCaseColNames = new ArrayList(request.getColNames().size()); + for (String colName : request.getColNames()) { + lowerCaseColNames.add(colName.toLowerCase()); + } try { - ColumnStatistics cs = getMS().getTableColumnStatistics( - dbName, tblName, request.getColNames()); + ColumnStatistics cs = getMS().getTableColumnStatistics(dbName, tblName, lowerCaseColNames); result = new TableStatsResult( cs == null ? Lists.newArrayList() : cs.getStatsObj()); } finally { @@ -4185,13 +4189,22 @@ public class HiveMetaStore extends Thrif @Override public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsRequest request) throws MetaException, NoSuchObjectException, TException { - String dbName = request.getDbName(), tblName = request.getTblName(); + String dbName = request.getDbName().toLowerCase(); + String tblName = request.getTblName().toLowerCase(); startFunction("get_partitions_statistics_req: db=" + dbName + " table=" + tblName); PartitionsStatsResult result = null; + List lowerCaseColNames = new ArrayList(request.getColNames().size()); + for (String colName : request.getColNames()) { + lowerCaseColNames.add(colName.toLowerCase()); + } + List lowerCasePartNames = new ArrayList(request.getPartNames().size()); + for (String partName : request.getPartNames()) { + lowerCasePartNames.add(lowerCaseConvertPartName(partName)); + } try { List stats = getMS().getPartitionColumnStatistics( - dbName, tblName, request.getPartNames(), request.getColNames()); + dbName, tblName, lowerCasePartNames, lowerCaseColNames); Map> map = new HashMap>(); for (ColumnStatistics stat : stats) { @@ -5579,11 +5592,23 @@ public class HiveMetaStore extends Thrif @Override public AggrStats get_aggr_stats_for(PartitionsStatsRequest request) throws NoSuchObjectException, MetaException, TException { + String dbName = request.getDbName().toLowerCase(); + String tblName = request.getTblName().toLowerCase(); startFunction("get_aggr_stats_for: db=" + request.getDbName() + " table=" + request.getTblName()); + + List lowerCaseColNames = new ArrayList(request.getColNames().size()); + for (String colName : request.getColNames()) { + lowerCaseColNames.add(colName.toLowerCase()); + } + List lowerCasePartNames = new ArrayList(request.getPartNames().size()); + for (String partName : request.getPartNames()) { + lowerCasePartNames.add(lowerCaseConvertPartName(partName)); + } AggrStats aggrStats = null; + try { - aggrStats = new AggrStats(getMS().get_aggr_stats_for(request.getDbName(), - request.getTblName(), request.getPartNames(), request.getColNames())); + aggrStats = new AggrStats(getMS().get_aggr_stats_for(dbName, tblName, lowerCasePartNames, + lowerCaseColNames)); return aggrStats; } finally { endFunction("get_partitions_statistics_req: ", aggrStats == null, null, request.getTblName()); Modified: hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original) +++ hive/branches/parquet/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Tue Mar 10 17:49:39 2015 @@ -32,9 +32,7 @@ import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.security.auth.login.LoginException; -import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -55,8 +53,6 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.hive.shims.Utils; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; /** @@ -250,41 +246,25 @@ public class Warehouse { return false; } final FileStatus stat; + final FileSystem fs; try { - stat = getFs(path).getFileStatus(path); + fs = getFs(path); + stat = fs.getFileStatus(path); + ShimLoader.getHadoopShims().checkFileAccess(fs, stat, FsAction.WRITE); + return true; } catch (FileNotFoundException fnfe){ // File named by path doesn't exist; nothing to validate. return true; } catch (Exception e) { // all other exceptions are considered as emanating from // unauthorized accesses - return false; - } - final UserGroupInformation ugi; - try { - ugi = Utils.getUGI(); - } catch (LoginException le) { - throw new IOException(le); - } - String user = ugi.getShortUserName(); - //check whether owner can delete - if (stat.getOwner().equals(user) && - stat.getPermission().getUserAction().implies(FsAction.WRITE)) { - return true; - } - //check whether group of the user can delete - if (stat.getPermission().getGroupAction().implies(FsAction.WRITE)) { - String[] groups = ugi.getGroupNames(); - if (ArrayUtils.contains(groups, stat.getGroup())) { - return true; + if (LOG.isDebugEnabled()) { + LOG.debug("Exception when checking if path (" + path + ")", e); } + return false; } - //check whether others can delete (uncommon case!!) - if (stat.getPermission().getOtherAction().implies(FsAction.WRITE)) { - return true; - } - return false; } + /* // NOTE: This is for generating the internal path name for partitions. Users // should always use the MetaStore API to get the path name for a partition. Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/SecureCmdDoAs.java Tue Mar 10 17:49:39 2015 @@ -38,6 +38,7 @@ import org.apache.hadoop.security.token. */ public class SecureCmdDoAs { private final Path tokenPath; + private final File tokenFile; public SecureCmdDoAs(HiveConf conf) throws HiveException, IOException{ // Get delegation token for user from filesystem and write the token along with @@ -46,8 +47,8 @@ public class SecureCmdDoAs { FileSystem fs = FileSystem.get(conf); Token fsToken = fs.getDelegationToken(uname); - File t = File.createTempFile("hive_hadoop_delegation_token", null); - tokenPath = new Path(t.toURI()); + tokenFile = File.createTempFile("hive_hadoop_delegation_token", null); + tokenPath = new Path(tokenFile.toURI()); //write credential with token to file Credentials cred = new Credentials(); @@ -60,4 +61,7 @@ public class SecureCmdDoAs { tokenPath.toUri().getPath()); } + public void close() { + tokenFile.delete(); + } } Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java Tue Mar 10 17:49:39 2015 @@ -95,6 +95,7 @@ public class MapredLocalTask extends Tas private ExecMapperContext execContext = null; private Process executor; + private SecureCmdDoAs secureDoAs; public MapredLocalTask() { super(); @@ -269,7 +270,7 @@ public class MapredLocalTask extends Tas //If kerberos security is enabled, and HS2 doAs is enabled, // then additional params need to be set so that the command is run as // intended user - SecureCmdDoAs secureDoAs = new SecureCmdDoAs(conf); + secureDoAs = new SecureCmdDoAs(conf); secureDoAs.addEnv(variables); } @@ -312,9 +313,12 @@ public class MapredLocalTask extends Tas return exitVal; } catch (Exception e) { - e.printStackTrace(); - LOG.error("Exception: " + e.getMessage()); + LOG.error("Exception: " + e, e); return (1); + } finally { + if (secureDoAs != null) { + secureDoAs.close(); + } } } Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java Tue Mar 10 17:49:39 2015 @@ -64,7 +64,7 @@ public class ParquetRecordReaderWrapper private boolean eof = false; private int schemaSize; private boolean skipTimestampConversion = false; - + private JobConf jobConf; private final ProjectionPusher projectionPusher; public ParquetRecordReaderWrapper( @@ -86,25 +86,26 @@ public class ParquetRecordReaderWrapper this.splitLen = oldSplit.getLength(); this.projectionPusher = pusher; - final ParquetInputSplit split = getSplit(oldSplit, oldJobConf); + jobConf = oldJobConf; + final ParquetInputSplit split = getSplit(oldSplit, jobConf); - TaskAttemptID taskAttemptID = TaskAttemptID.forName(oldJobConf.get(IOConstants.MAPRED_TASK_ID)); + TaskAttemptID taskAttemptID = TaskAttemptID.forName(jobConf.get(IOConstants.MAPRED_TASK_ID)); if (taskAttemptID == null) { taskAttemptID = new TaskAttemptID(); } - setFilter(oldJobConf); + setFilter(jobConf); // create a TaskInputOutputContext - Configuration conf = oldJobConf; + Configuration conf = jobConf; if (skipTimestampConversion ^ HiveConf.getBoolVar( conf, HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION)) { conf = new JobConf(oldJobConf); HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION, skipTimestampConversion); } - final TaskAttemptContext taskContext = ContextUtil.newTaskAttemptContext(conf, taskAttemptID); + final TaskAttemptContext taskContext = ContextUtil.newTaskAttemptContext(conf, taskAttemptID); if (split != null) { try { realReader = newInputFormat.createRecordReader(split, taskContext); @@ -236,14 +237,14 @@ public class ParquetRecordReaderWrapper ParquetInputSplit split; if (oldSplit instanceof FileSplit) { final Path finalPath = ((FileSplit) oldSplit).getPath(); - final JobConf cloneJob = projectionPusher.pushProjectionsAndFilters(conf, finalPath.getParent()); + jobConf = projectionPusher.pushProjectionsAndFilters(conf, finalPath.getParent()); - final ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(cloneJob, finalPath); + final ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(jobConf, finalPath); final List blocks = parquetMetadata.getBlocks(); final FileMetaData fileMetaData = parquetMetadata.getFileMetaData(); final ReadContext readContext = new DataWritableReadSupport() - .init(cloneJob, fileMetaData.getKeyValueMetaData(), fileMetaData.getSchema()); + .init(jobConf, fileMetaData.getKeyValueMetaData(), fileMetaData.getSchema()); schemaSize = MessageTypeParser.parseMessageType(readContext.getReadSupportMetadata() .get(DataWritableReadSupport.HIVE_SCHEMA_KEY)).getFieldCount(); final List splitGroup = new ArrayList(); Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Tue Mar 10 17:49:39 2015 @@ -111,7 +111,7 @@ public class ConvertJoinMapJoin implemen } if (parentOp instanceof ReduceSinkOperator) { - ReduceSinkOperator rs = (ReduceSinkOperator)parentOp; + ReduceSinkOperator rs = (ReduceSinkOperator) parentOp; estimatedBuckets = (estimatedBuckets < rs.getConf().getNumReducers()) ? rs.getConf().getNumReducers() : estimatedBuckets; } @@ -133,10 +133,10 @@ public class ConvertJoinMapJoin implemen if (retval == null) { return retval; } else { - // only case is full outer join with SMB enabled which is not possible. Convert to regular - // join. - convertJoinSMBJoin(joinOp, context, 0, 0, false, false); - return null; + // only case is full outer join with SMB enabled which is not possible. Convert to regular + // join. + convertJoinSMBJoin(joinOp, context, 0, 0, false, false); + return null; } } @@ -160,8 +160,10 @@ public class ConvertJoinMapJoin implemen } MapJoinOperator mapJoinOp = convertJoinMapJoin(joinOp, context, mapJoinConversionPos); - // map join operator by default has no bucket cols - mapJoinOp.setOpTraits(new OpTraits(null, -1, null)); + // map join operator by default has no bucket cols and num of reduce sinks + // reduced by 1 + mapJoinOp + .setOpTraits(new OpTraits(null, -1, null, joinOp.getOpTraits().getNumReduceSinks())); mapJoinOp.setStatistics(joinOp.getStatistics()); // propagate this change till the next RS for (Operator childOp : mapJoinOp.getChildOperators()) { @@ -176,7 +178,8 @@ public class ConvertJoinMapJoin implemen TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException { // we cannot convert to bucket map join, we cannot convert to // map join either based on the size. Check if we can convert to SMB join. - if (context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false) { + if ((context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false) + || (joinOp.getOpTraits().getNumReduceSinks() >= 2)) { convertJoinSMBJoin(joinOp, context, 0, 0, false, false); return null; } @@ -221,7 +224,7 @@ public class ConvertJoinMapJoin implemen convertJoinSMBJoin(joinOp, context, pos, 0, false, false); } return null; -} + } // replaces the join operator with a new CommonJoinOperator, removes the // parent reduce sinks @@ -240,9 +243,9 @@ public class ConvertJoinMapJoin implemen new MapJoinDesc( MapJoinProcessor.getKeys(joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(), joinOp).getSecond(), - null, joinDesc.getExprs(), null, null, - joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(), - joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null); + null, joinDesc.getExprs(), null, null, + joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(), + joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null); mapJoinDesc.setNullSafes(joinDesc.getNullSafes()); mapJoinDesc.setFilterMap(joinDesc.getFilterMap()); mapJoinDesc.resetOrder(); @@ -251,9 +254,9 @@ public class ConvertJoinMapJoin implemen CommonMergeJoinOperator mergeJoinOp = (CommonMergeJoinOperator) OperatorFactory.get(new CommonMergeJoinDesc(numBuckets, isSubQuery, mapJoinConversionPos, mapJoinDesc), joinOp.getSchema()); - OpTraits opTraits = - new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp.getOpTraits() - .getSortCols()); + int numReduceSinks = joinOp.getOpTraits().getNumReduceSinks(); + OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp + .getOpTraits().getSortCols(), numReduceSinks); mergeJoinOp.setOpTraits(opTraits); mergeJoinOp.setStatistics(joinOp.getStatistics()); @@ -289,8 +292,7 @@ public class ConvertJoinMapJoin implemen if (adjustParentsChildren) { mergeJoinOp.getConf().setGenJoinKeys(true); - List> newParentOpList = - new ArrayList>(); + List> newParentOpList = new ArrayList>(); for (Operator parentOp : mergeJoinOp.getParentOperators()) { for (Operator grandParentOp : parentOp.getParentOperators()) { grandParentOp.getChildOperators().remove(parentOp); @@ -328,7 +330,8 @@ public class ConvertJoinMapJoin implemen if (currentOp instanceof ReduceSinkOperator) { return; } - currentOp.setOpTraits(new OpTraits(null, -1, null)); + currentOp.setOpTraits(new OpTraits(null, -1, null, + currentOp.getOpTraits().getNumReduceSinks())); for (Operator childOp : currentOp.getChildOperators()) { if ((childOp instanceof ReduceSinkOperator) || (childOp instanceof GroupByOperator)) { break; @@ -351,7 +354,7 @@ public class ConvertJoinMapJoin implemen // we can set the traits for this join operator OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), - tezBucketJoinProcCtx.getNumBuckets(), null); + tezBucketJoinProcCtx.getNumBuckets(), null, joinOp.getOpTraits().getNumReduceSinks()); mapJoinOp.setOpTraits(opTraits); mapJoinOp.setStatistics(joinOp.getStatistics()); setNumberOfBucketsOnChildren(mapJoinOp); @@ -377,8 +380,7 @@ public class ConvertJoinMapJoin implemen ReduceSinkOperator bigTableRS = (ReduceSinkOperator) joinOp.getParentOperators().get(bigTablePosition); - int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits() - .getNumBuckets(); + int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits().getNumBuckets(); // the sort and bucket cols have to match on both sides for this // transformation of the join operation @@ -425,13 +427,12 @@ public class ConvertJoinMapJoin implemen } /* - * If the parent reduce sink of the big table side has the same emit key cols - * as its parent, we can create a bucket map join eliminating the reduce sink. + * If the parent reduce sink of the big table side has the same emit key cols as its parent, we + * can create a bucket map join eliminating the reduce sink. */ private boolean checkConvertJoinBucketMapJoin(JoinOperator joinOp, OptimizeTezProcContext context, int bigTablePosition, - TezBucketJoinProcCtx tezBucketJoinProcCtx) - throws SemanticException { + TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException { // bail on mux-operator because mux operator masks the emit keys of the // constituent reduce sinks if (!(joinOp.getParentOperators().get(0) instanceof ReduceSinkOperator)) { @@ -453,8 +454,8 @@ public class ConvertJoinMapJoin implemen } /* - * this is the case when the big table is a sub-query and is probably - * already bucketed by the join column in say a group by operation + * this is the case when the big table is a sub-query and is probably already bucketed by the + * join column in say a group by operation */ boolean isSubQuery = false; if (numBuckets < 0) { @@ -484,10 +485,16 @@ public class ConvertJoinMapJoin implemen int colCount = 0; // parent op is guaranteed to have a single list because it is a reduce sink for (String colName : parentColNames.get(0)) { + if (listBucketCols.size() <= colCount) { + // can happen with virtual columns. RS would add the column to its output columns + // but it would not exist in the grandparent output columns or exprMap. + return false; + } // all columns need to be at least a subset of the parentOfParent's bucket cols ExprNodeDesc exprNodeDesc = colExprMap.get(colName); if (exprNodeDesc instanceof ExprNodeColumnDesc) { - if (((ExprNodeColumnDesc)exprNodeDesc).getColumn().equals(listBucketCols.get(colCount))) { + if (((ExprNodeColumnDesc) exprNodeDesc).getColumn() + .equals(listBucketCols.get(colCount))) { colCount++; } else { break; @@ -557,14 +564,13 @@ public class ConvertJoinMapJoin implemen Statistics currInputStat = parentOp.getStatistics(); if (currInputStat == null) { - LOG.warn("Couldn't get statistics from: "+parentOp); + LOG.warn("Couldn't get statistics from: " + parentOp); return -1; } long inputSize = currInputStat.getDataSize(); - if ((bigInputStat == null) || - ((bigInputStat != null) && - (inputSize > bigInputStat.getDataSize()))) { + if ((bigInputStat == null) + || ((bigInputStat != null) && (inputSize > bigInputStat.getDataSize()))) { if (bigTableFound) { // cannot convert to map join; we've already chosen a big table @@ -634,11 +640,11 @@ public class ConvertJoinMapJoin implemen } } - //can safely convert the join to a map join. + // can safely convert the join to a map join. MapJoinOperator mapJoinOp = MapJoinProcessor.convertJoinOpMapJoinOp(context.conf, joinOp, - joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(), - joinOp.getConf().getMapAliases(), bigTablePosition, true); + joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(), + joinOp.getConf().getMapAliases(), bigTablePosition, true); Operator parentBigTableOp = mapJoinOp.getParentOperators().get(bigTablePosition); @@ -662,7 +668,7 @@ public class ConvertJoinMapJoin implemen parentBigTableOp.getParentOperators().get(0)); } parentBigTableOp.getParentOperators().get(0).removeChild(parentBigTableOp); - for (Operator op : mapJoinOp.getParentOperators()) { + for (Operatorop : mapJoinOp.getParentOperators()) { if (!(op.getChildOperators().contains(mapJoinOp))) { op.getChildOperators().add(mapJoinOp); } @@ -676,7 +682,7 @@ public class ConvertJoinMapJoin implemen private boolean hasDynamicPartitionBroadcast(Operator parent) { boolean hasDynamicPartitionPruning = false; - for (Operator op: parent.getChildOperators()) { + for (Operator op : parent.getChildOperators()) { while (op != null) { if (op instanceof AppMasterEventOperator && op.getConf() instanceof DynamicPruningEventDesc) { // found dynamic partition pruning operator Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java Tue Mar 10 17:49:39 2015 @@ -82,7 +82,7 @@ public class OpTraitsRulesProcFactory { } /* - * Reduce sink operator is the de-facto operator + * Reduce sink operator is the de-facto operator * for determining keyCols (emit keys of a map phase) */ public static class ReduceSinkRule implements NodeProcessor { @@ -106,24 +106,25 @@ public class OpTraitsRulesProcFactory { List> listBucketCols = new ArrayList>(); listBucketCols.add(bucketCols); int numBuckets = -1; + int numReduceSinks = 1; OpTraits parentOpTraits = rs.getParentOperators().get(0).getConf().getOpTraits(); if (parentOpTraits != null) { numBuckets = parentOpTraits.getNumBuckets(); + numReduceSinks += parentOpTraits.getNumReduceSinks(); } - OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols); + OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols, numReduceSinks); rs.setOpTraits(opTraits); return null; } } /* - * Table scan has the table object and pruned partitions that has information such as - * bucketing, sorting, etc. that is used later for optimization. + * Table scan has the table object and pruned partitions that has information + * such as bucketing, sorting, etc. that is used later for optimization. */ public static class TableScanRule implements NodeProcessor { - public boolean checkBucketedTable(Table tbl, - ParseContext pGraphContext, + public boolean checkBucketedTable(Table tbl, ParseContext pGraphContext, PrunedPartitionList prunedParts) throws SemanticException { if (tbl.isPartitioned()) { @@ -131,9 +132,11 @@ public class OpTraitsRulesProcFactory { // construct a mapping of (Partition->bucket file names) and (Partition -> bucket number) if (!partitions.isEmpty()) { for (Partition p : partitions) { - List fileNames = - AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(), pGraphContext); - // The number of files for the table should be same as number of buckets. + List fileNames = + AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(), + pGraphContext); + // The number of files for the table should be same as number of + // buckets. int bucketCount = p.getBucketCount(); if (fileNames.size() != 0 && fileNames.size() != bucketCount) { @@ -143,8 +146,9 @@ public class OpTraitsRulesProcFactory { } } else { - List fileNames = - AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(), pGraphContext); + List fileNames = + AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(), + pGraphContext); Integer num = new Integer(tbl.getNumBuckets()); // The number of files for the table should be same as number of buckets. @@ -183,7 +187,8 @@ public class OpTraitsRulesProcFactory { } sortedColsList.add(sortCols); } - OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList); + // num reduce sinks hardcoded to 0 because TS has no parents + OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList, 0); ts.setOpTraits(opTraits); return null; } @@ -208,8 +213,13 @@ public class OpTraitsRulesProcFactory { } List> listBucketCols = new ArrayList>(); + int numReduceSinks = 0; + OpTraits parentOpTraits = gbyOp.getParentOperators().get(0).getOpTraits(); + if (parentOpTraits != null) { + numReduceSinks = parentOpTraits.getNumReduceSinks(); + } listBucketCols.add(gbyKeys); - OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols); + OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols, numReduceSinks); gbyOp.setOpTraits(opTraits); return null; } @@ -217,8 +227,8 @@ public class OpTraitsRulesProcFactory { public static class SelectRule implements NodeProcessor { - public List> getConvertedColNames(List> parentColNames, - SelectOperator selOp) { + public List> getConvertedColNames( + List> parentColNames, SelectOperator selOp) { List> listBucketCols = new ArrayList>(); if (selOp.getColumnExprMap() != null) { if (parentColNames != null) { @@ -244,8 +254,8 @@ public class OpTraitsRulesProcFactory { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { - SelectOperator selOp = (SelectOperator)nd; - List> parentBucketColNames = + SelectOperator selOp = (SelectOperator) nd; + List> parentBucketColNames = selOp.getParentOperators().get(0).getOpTraits().getBucketColNames(); List> listBucketCols = null; @@ -254,18 +264,21 @@ public class OpTraitsRulesProcFactory { if (parentBucketColNames != null) { listBucketCols = getConvertedColNames(parentBucketColNames, selOp); } - List> parentSortColNames = selOp.getParentOperators().get(0).getOpTraits() - .getSortCols(); + List> parentSortColNames = + selOp.getParentOperators().get(0).getOpTraits().getSortCols(); if (parentSortColNames != null) { listSortCols = getConvertedColNames(parentSortColNames, selOp); } } int numBuckets = -1; - if (selOp.getParentOperators().get(0).getOpTraits() != null) { - numBuckets = selOp.getParentOperators().get(0).getOpTraits().getNumBuckets(); + int numReduceSinks = 0; + OpTraits parentOpTraits = selOp.getParentOperators().get(0).getOpTraits(); + if (parentOpTraits != null) { + numBuckets = parentOpTraits.getNumBuckets(); + numReduceSinks = parentOpTraits.getNumReduceSinks(); } - OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols); + OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols, numReduceSinks); selOp.setOpTraits(opTraits); return null; } @@ -276,26 +289,31 @@ public class OpTraitsRulesProcFactory { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { - JoinOperator joinOp = (JoinOperator)nd; + JoinOperator joinOp = (JoinOperator) nd; List> bucketColsList = new ArrayList>(); List> sortColsList = new ArrayList>(); byte pos = 0; + int numReduceSinks = 0; // will be set to the larger of the parents for (Operator parentOp : joinOp.getParentOperators()) { if (!(parentOp instanceof ReduceSinkOperator)) { // can be mux operator break; } - ReduceSinkOperator rsOp = (ReduceSinkOperator)parentOp; + ReduceSinkOperator rsOp = (ReduceSinkOperator) parentOp; if (rsOp.getOpTraits() == null) { ReduceSinkRule rsRule = new ReduceSinkRule(); rsRule.process(rsOp, stack, procCtx, nodeOutputs); } - bucketColsList.add(getOutputColNames(joinOp, rsOp.getOpTraits().getBucketColNames(), pos)); - sortColsList.add(getOutputColNames(joinOp, rsOp.getOpTraits().getSortCols(), pos)); + OpTraits parentOpTraits = rsOp.getOpTraits(); + bucketColsList.add(getOutputColNames(joinOp, parentOpTraits.getBucketColNames(), pos)); + sortColsList.add(getOutputColNames(joinOp, parentOpTraits.getSortCols(), pos)); + if (parentOpTraits.getNumReduceSinks() > numReduceSinks) { + numReduceSinks = parentOpTraits.getNumReduceSinks(); + } pos++; } - joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList)); + joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList, numReduceSinks)); return null; } @@ -311,7 +329,7 @@ public class OpTraitsRulesProcFactory { for (String colName : colNames) { for (ExprNodeDesc exprNode : joinOp.getConf().getExprs().get(pos)) { if (exprNode instanceof ExprNodeColumnDesc) { - if(((ExprNodeColumnDesc)(exprNode)).getColumn().equals(colName)) { + if (((ExprNodeColumnDesc) (exprNode)).getColumn().equals(colName)) { for (Entry entry : joinOp.getColumnExprMap().entrySet()) { if (entry.getValue().isSame(exprNode)) { bucketColNames.add(entry.getKey()); @@ -338,20 +356,30 @@ public class OpTraitsRulesProcFactory { } /* - * When we have operators that have multiple parents, it is not - * clear which parent's traits we need to propagate forward. + * When we have operators that have multiple parents, it is not clear which + * parent's traits we need to propagate forward. */ public static class MultiParentRule implements NodeProcessor { @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { - OpTraits opTraits = new OpTraits(null, -1, null); @SuppressWarnings("unchecked") - Operator operator = (Operator)nd; + Operator operator = (Operator) nd; + + int numReduceSinks = 0; + for (Operator parentOp : operator.getParentOperators()) { + if (parentOp.getOpTraits() == null) { + continue; + } + if (parentOp.getOpTraits().getNumReduceSinks() > numReduceSinks) { + numReduceSinks = parentOp.getOpTraits().getNumReduceSinks(); + } + } + OpTraits opTraits = new OpTraits(null, -1, null, numReduceSinks); operator.setOpTraits(opTraits); return null; - } + } } public static NodeProcessor getTableScanRule() { @@ -361,7 +389,7 @@ public class OpTraitsRulesProcFactory { public static NodeProcessor getReduceSinkRule() { return new ReduceSinkRule(); } - + public static NodeProcessor getSelectRule() { return new SelectRule(); } Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java Tue Mar 10 17:49:39 2015 @@ -103,7 +103,7 @@ public class SparkMapJoinOptimizer imple } // we can set the traits for this join operator - OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null); + OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null, joinOp.getOpTraits().getNumReduceSinks()); mapJoinOp.setOpTraits(opTraits); mapJoinOp.setStatistics(joinOp.getStatistics()); setNumberOfBucketsOnChildren(mapJoinOp); Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java Tue Mar 10 17:49:39 2015 @@ -718,7 +718,7 @@ public class CalcitePlanner extends Sema hepPgmBldr.addRuleInstance(ReduceExpressionsRule.JOIN_INSTANCE); hepPgmBldr.addRuleInstance(ReduceExpressionsRule.FILTER_INSTANCE); hepPgmBldr.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE); - hepPgmBldr.addRuleInstance(ProjectRemoveRule.NAME_CALC_INSTANCE); + hepPgmBldr.addRuleInstance(ProjectRemoveRule.INSTANCE); hepPgmBldr.addRuleInstance(UnionMergeRule.INSTANCE); hepPgm = hepPgmBldr.build(); @@ -792,7 +792,7 @@ public class CalcitePlanner extends Sema RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY, HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY, RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY, - HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY, true); + HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY); basePlan = fieldTrimmer.trim(basePlan); // 5. Rerun PPD through Project as column pruning would have introduced DT Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Mar 10 17:49:39 2015 @@ -555,7 +555,7 @@ public class TypeCheckProcFactory { private static ExprNodeDesc toExprNodeDesc(ColumnInfo colInfo) { ObjectInspector inspector = colInfo.getObjectInspector(); - if (inspector instanceof ConstantObjectInspector && + if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector; Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue(); @@ -994,8 +994,15 @@ public class TypeCheckProcFactory { .getText()); // NOTE: tableAlias must be a valid non-ambiguous table alias, // because we've checked that in TOK_TABLE_OR_COL's process method. - ColumnInfo colInfo = input.get(tableAlias, ((ExprNodeConstantDesc) nodeOutputs[1]).getValue() - .toString()); + String colName; + if (nodeOutputs[1] instanceof ExprNodeConstantDesc) { + colName = ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString(); + } else if (nodeOutputs[1] instanceof ExprNodeColumnDesc) { + colName = ((ExprNodeColumnDesc)nodeOutputs[1]).getColumn(); + } else { + throw new SemanticException("Unexpected ExprNode : " + nodeOutputs[1]); + } + ColumnInfo colInfo = input.get(tableAlias, colName); if (colInfo == null) { ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr); Modified: hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java URL: http://svn.apache.org/viewvc/hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java?rev=1665637&r1=1665636&r2=1665637&view=diff ============================================================================== --- hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java (original) +++ hive/branches/parquet/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java Tue Mar 10 17:49:39 2015 @@ -25,11 +25,14 @@ public class OpTraits { List> bucketColNames; List> sortColNames; int numBuckets; + int numReduceSinks; - public OpTraits(List> bucketColNames, int numBuckets, List> sortColNames) { + public OpTraits(List> bucketColNames, int numBuckets, + List> sortColNames, int numReduceSinks) { this.bucketColNames = bucketColNames; this.numBuckets = numBuckets; this.sortColNames = sortColNames; + this.numReduceSinks = numReduceSinks; } public List> getBucketColNames() { @@ -55,4 +58,12 @@ public class OpTraits { public List> getSortCols() { return sortColNames; } + + public void setNumReduceSinks(int numReduceSinks) { + this.numReduceSinks = numReduceSinks; + } + + public int getNumReduceSinks() { + return this.numReduceSinks; + } }