hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject hive git commit: HIVE-16771: Schematool should use MetastoreSchemaInfo to get the metastore schema version from database (Vihang Karajgaonkar, reviewed by Naveen Gangam, Sergio Pena)
Date Wed, 31 May 2017 18:50:50 GMT
Repository: hive
Updated Branches:
  refs/heads/master 396cba458 -> cb815dcf9


HIVE-16771: Schematool should use MetastoreSchemaInfo to get the metastore schema version from database (Vihang Karajgaonkar, reviewed by Naveen Gangam, Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb815dcf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb815dcf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb815dcf

Branch: refs/heads/master
Commit: cb815dcf96030cb48b4d8ab89895ff3141f7dbca
Parents: 396cba4
Author: Vihang Karajgaonkar <vihang@cloudera.com>
Authored: Wed May 31 13:50:14 2017 -0500
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Wed May 31 13:50:14 2017 -0500

----------------------------------------------------------------------
 .../apache/hive/beeline/HiveSchemaHelper.java   | 581 -----------------
 .../org/apache/hive/beeline/HiveSchemaTool.java |  79 +--
 .../apache/hive/beeline/TestHiveSchemaTool.java |   1 +
 .../org/apache/hive/beeline/TestSchemaTool.java |   6 +-
 .../hive/metastore/IMetaStoreSchemaInfo.java    |  15 +
 .../hive/metastore/MetaStoreSchemaInfo.java     |  33 +
 .../hive/metastore/tools/HiveSchemaHelper.java  | 640 +++++++++++++++++++
 7 files changed, 719 insertions(+), 636 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
deleted file mode 100644
index a4ecc08..0000000
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.IllegalFormatException;
-import java.util.List;
-
-public class HiveSchemaHelper {
-  public static final String DB_DERBY = "derby";
-  public static final String DB_HIVE = "hive";
-  public static final String DB_MSSQL = "mssql";
-  public static final String DB_MYSQL = "mysql";
-  public static final String DB_POSTGRACE = "postgres";
-  public static final String DB_ORACLE = "oracle";
-
-  /***
-   * Get JDBC connection to metastore db
-   *
-   * @param userName metastore connection username
-   * @param password metastore connection password
-   * @param printInfo print connection parameters
-   * @param hiveConf hive config object
-   * @return metastore connection object
-   * @throws org.apache.hadoop.hive.metastore.api.MetaException
-   */
-  public static Connection getConnectionToMetastore(String userName,
-      String password, String url, String driver, boolean printInfo,
-      HiveConf hiveConf)
-      throws HiveMetaException {
-    try {
-      url = url == null ? getValidConfVar(
-        HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf) : url;
-      driver = driver == null ? getValidConfVar(
-        HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver;
-      if (printInfo) {
-        System.out.println("Metastore connection URL:\t " + url);
-        System.out.println("Metastore Connection Driver :\t " + driver);
-        System.out.println("Metastore connection User:\t " + userName);
-      }
-      if ((userName == null) || userName.isEmpty()) {
-        throw new HiveMetaException("UserName empty ");
-      }
-
-      // load required JDBC driver
-      Class.forName(driver);
-
-      // Connect using the JDBC URL and user/pass from conf
-      return DriverManager.getConnection(url, userName, password);
-    } catch (IOException e) {
-      throw new HiveMetaException("Failed to get schema version.", e);
-    } catch (SQLException e) {
-      throw new HiveMetaException("Failed to get schema version.", e);
-    } catch (ClassNotFoundException e) {
-      throw new HiveMetaException("Failed to load driver", e);
-    }
-  }
-
-  public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf)
-      throws IOException {
-    String confVarStr = hiveConf.get(confVar.varname);
-    if (confVarStr == null || confVarStr.isEmpty()) {
-      throw new IOException("Empty " + confVar.varname);
-    }
-    return confVarStr.trim();
-  }
-
-  public interface NestedScriptParser {
-
-    public enum CommandType {
-      PARTIAL_STATEMENT,
-      TERMINATED_STATEMENT,
-      COMMENT
-    }
-
-    static final String DEFAULT_DELIMITER = ";";
-    static final String DEFAULT_QUOTE = "\"";
-
-    /**
-     * Find the type of given command
-     *
-     * @param dbCommand
-     * @return
-     */
-    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
-
-    /**
-     * Parse the DB specific nesting format and extract the inner script name if any
-     *
-     * @param dbCommand command from parent script
-     * @return
-     * @throws IllegalFormatException
-     */
-    public String getScriptName(String dbCommand) throws IllegalArgumentException;
-
-    /**
-     * Find if the given command is a nested script execution
-     *
-     * @param dbCommand
-     * @return
-     */
-    public boolean isNestedScript(String dbCommand);
-
-    /**
-     * Find if the given command should not be passed to DB
-     *
-     * @param dbCommand
-     * @return
-     */
-    public boolean isNonExecCommand(String dbCommand);
-
-    /**
-     * Get the SQL statement delimiter
-     *
-     * @return
-     */
-    public String getDelimiter();
-
-    /**
-     * Get the SQL indentifier quotation character
-     *
-     * @return
-     */
-    public String getQuoteCharacter();
-
-    /**
-     * Clear any client specific tags
-     *
-     * @return
-     */
-    public String cleanseCommand(String dbCommand);
-
-    /**
-     * Does the DB required table/column names quoted
-     *
-     * @return
-     */
-    public boolean needsQuotedIdentifier();
-
-    /**
-     * Flatten the nested upgrade script into a buffer
-     *
-     * @param scriptDir  upgrade script directory
-     * @param scriptFile upgrade script file
-     * @return string of sql commands
-     */
-    public String buildCommand(String scriptDir, String scriptFile)
-        throws IllegalFormatException, IOException;
-
-    /**
-     * Flatten the nested upgrade script into a buffer
-     *
-     * @param scriptDir  upgrade script directory
-     * @param scriptFile upgrade script file
-     * @param fixQuotes whether to replace quote characters
-     * @return string of sql commands
-     */
-    public String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes)
-        throws IllegalFormatException, IOException;
-  }
-
-  /***
-   * Base implementation of NestedScriptParser
-   * abstractCommandParser.
-   *
-   */
-  private static abstract class AbstractCommandParser implements NestedScriptParser {
-    private List<String> dbOpts;
-    private String msUsername;
-    private String msPassword;
-    private HiveConf hiveConf;
-
-    public AbstractCommandParser(String dbOpts, String msUsername, String msPassword,
-        HiveConf hiveConf) {
-      setDbOpts(dbOpts);
-      this.msUsername = msUsername;
-      this.msPassword = msPassword;
-      this.hiveConf = hiveConf;
-    }
-
-    @Override
-    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
-      if (dbCommand == null || dbCommand.isEmpty()) {
-        throw new IllegalArgumentException("invalid command line " + dbCommand);
-      }
-      dbCommand = dbCommand.trim();
-      if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) {
-        return false;
-      } else {
-        return true;
-      }
-    }
-
-    @Override
-    public boolean isNonExecCommand(String dbCommand) {
-      return (dbCommand.startsWith("--") || dbCommand.startsWith("#"));
-    }
-
-    @Override
-    public String getDelimiter() {
-      return DEFAULT_DELIMITER;
-    }
-
-    @Override
-    public String getQuoteCharacter() {
-      return DEFAULT_QUOTE;
-    }
-
-
-    @Override
-    public String cleanseCommand(String dbCommand) {
-      // strip off the delimiter
-      if (dbCommand.endsWith(getDelimiter())) {
-        dbCommand = dbCommand.substring(0,
-            dbCommand.length() - getDelimiter().length());
-      }
-      return dbCommand;
-    }
-
-    @Override
-    public boolean needsQuotedIdentifier() {
-      return false;
-    }
-
-    @Override
-    public String buildCommand(
-      String scriptDir, String scriptFile) throws IllegalFormatException, IOException {
-      return buildCommand(scriptDir, scriptFile, false);
-    }
-
-    @Override
-    public String buildCommand(
-      String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException {
-      BufferedReader bfReader =
-          new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile));
-      String currLine;
-      StringBuilder sb = new StringBuilder();
-      String currentCommand = null;
-      while ((currLine = bfReader.readLine()) != null) {
-        currLine = currLine.trim();
-
-        if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) {
-          currLine = currLine.replace("\\\"", getQuoteCharacter());
-        }
-
-        if (currLine.isEmpty()) {
-          continue; // skip empty lines
-        }
-
-        if (currentCommand == null) {
-          currentCommand = currLine;
-        } else {
-          currentCommand = currentCommand + " " + currLine;
-        }
-        if (isPartialCommand(currLine)) {
-          // if its a partial line, continue collecting the pieces
-          continue;
-        }
-
-        // if this is a valid executable command then add it to the buffer
-        if (!isNonExecCommand(currentCommand)) {
-          currentCommand = cleanseCommand(currentCommand);
-          if (isNestedScript(currentCommand)) {
-            // if this is a nested sql script then flatten it
-            String currScript = getScriptName(currentCommand);
-            sb.append(buildCommand(scriptDir, currScript));
-          } else {
-            // Now we have a complete statement, process it
-            // write the line to buffer
-            sb.append(currentCommand);
-            sb.append(System.getProperty("line.separator"));
-          }
-        }
-        currentCommand = null;
-      }
-      bfReader.close();
-      return sb.toString();
-    }
-
-    private void setDbOpts(String dbOpts) {
-      if (dbOpts != null) {
-        this.dbOpts = Lists.newArrayList(dbOpts.split(","));
-      } else {
-        this.dbOpts = Lists.newArrayList();
-      }
-    }
-
-    protected List<String> getDbOpts() {
-      return dbOpts;
-    }
-
-    protected String getMsUsername() {
-      return msUsername;
-    }
-
-    protected String getMsPassword() {
-      return msPassword;
-    }
-
-    protected HiveConf getHiveConf() {
-      return hiveConf;
-    }
-  }
-
-  // Derby commandline parser
-  public static class DerbyCommandParser extends AbstractCommandParser {
-    private static final String DERBY_NESTING_TOKEN = "RUN";
-
-    public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
-        HiveConf hiveConf) {
-      super(dbOpts, msUsername, msPassword, hiveConf);
-    }
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
-      if (!isNestedScript(dbCommand)) {
-        throw new IllegalArgumentException("Not a script format " + dbCommand);
-      }
-      String[] tokens = dbCommand.split(" ");
-      if (tokens.length != 2) {
-        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-      }
-      return tokens[1].replace(";", "").replaceAll("'", "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      // Derby script format is RUN '<file>'
-     return dbCommand.startsWith(DERBY_NESTING_TOKEN);
-    }
-  }
-
-  // Derby commandline parser
-  public static class HiveCommandParser extends AbstractCommandParser {
-    private static String HIVE_NESTING_TOKEN = "SOURCE";
-    private final NestedScriptParser nestedDbCommandParser;
-
-    public HiveCommandParser(String dbOpts, String msUsername, String msPassword,
-        HiveConf hiveConf, String metaDbType) {
-      super(dbOpts, msUsername, msPassword, hiveConf);
-      nestedDbCommandParser = getDbCommandParser(metaDbType);
-    }
-
-    @Override
-    public String getQuoteCharacter() {
-      return nestedDbCommandParser.getQuoteCharacter();
-    }
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
-      if (!isNestedScript(dbCommand)) {
-        throw new IllegalArgumentException("Not a script format " + dbCommand);
-      }
-      String[] tokens = dbCommand.split(" ");
-      if (tokens.length != 2) {
-        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-      }
-      return tokens[1].replace(";", "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-     return dbCommand.startsWith(HIVE_NESTING_TOKEN);
-    }
-  }
-
-  // MySQL parser
-  public static class MySqlCommandParser extends AbstractCommandParser {
-    private static final String MYSQL_NESTING_TOKEN = "SOURCE";
-    private static final String DELIMITER_TOKEN = "DELIMITER";
-    private String delimiter = DEFAULT_DELIMITER;
-
-    public MySqlCommandParser(String dbOpts, String msUsername, String msPassword,
-        HiveConf hiveConf) {
-      super(dbOpts, msUsername, msPassword, hiveConf);
-    }
-
-    @Override
-    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
-      boolean isPartial = super.isPartialCommand(dbCommand);
-      // if this is a delimiter directive, reset our delimiter
-      if (dbCommand.startsWith(DELIMITER_TOKEN)) {
-        String[] tokens = dbCommand.split(" ");
-        if (tokens.length != 2) {
-          throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-        }
-        delimiter = tokens[1];
-      }
-      return isPartial;
-    }
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-      String[] tokens = dbCommand.split(" ");
-      if (tokens.length != 2) {
-        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-      }
-      // remove ending ';'
-      return tokens[1].replace(";", "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
-    }
-
-    @Override
-    public String getDelimiter() {
-      return delimiter;
-    }
-
-    @Override
-    public String getQuoteCharacter() {
-      return "`";
-    }
-
-    @Override
-    public boolean isNonExecCommand(String dbCommand) {
-      return super.isNonExecCommand(dbCommand) ||
-          (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) ||
-          dbCommand.startsWith(DELIMITER_TOKEN);
-    }
-
-    @Override
-    public String cleanseCommand(String dbCommand) {
-      return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
-    }
-
-  }
-
-  // Postgres specific parser
-  public static class PostgresCommandParser extends AbstractCommandParser {
-    private static final String POSTGRES_NESTING_TOKEN = "\\i";
-    @VisibleForTesting
-    public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
-    @VisibleForTesting
-    public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
-
-    public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
-        HiveConf hiveConf) {
-      super(dbOpts, msUsername, msPassword, hiveConf);
-    }
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-      String[] tokens = dbCommand.split(" ");
-      if (tokens.length != 2) {
-        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-      }
-      // remove ending ';'
-      return tokens[1].replace(";", "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
-    }
-
-    @Override
-    public boolean needsQuotedIdentifier() {
-      return true;
-    }
-
-    @Override
-    public boolean isNonExecCommand(String dbCommand) {
-      // Skip "standard_conforming_strings" command which is read-only in older
-      // Postgres versions like 8.1
-      // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html
-      if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) {
-        if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) {
-          return true;
-        }
-      }
-      return super.isNonExecCommand(dbCommand);
-    }
-  }
-
-  //Oracle specific parser
-  public static class OracleCommandParser extends AbstractCommandParser {
-    private static final String ORACLE_NESTING_TOKEN = "@";
-
-    public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
-        HiveConf hiveConf) {
-      super(dbOpts, msUsername, msPassword, hiveConf);
-    }
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-      if (!isNestedScript(dbCommand)) {
-        throw new IllegalArgumentException("Not a nested script format " + dbCommand);
-      }
-      // remove ending ';' and starting '@'
-      return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
-    }
-  }
-
-  //MSSQL specific parser
-  public static class MSSQLCommandParser extends AbstractCommandParser {
-    private static final String MSSQL_NESTING_TOKEN = ":r";
-
-    public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
-        HiveConf hiveConf) {
-      super(dbOpts, msUsername, msPassword, hiveConf);
-    }
-
-    @Override
-    public String getScriptName(String dbCommand) throws IllegalArgumentException {
-      String[] tokens = dbCommand.split(" ");
-      if (tokens.length != 2) {
-        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
-      }
-      return tokens[1];
-    }
-
-    @Override
-    public boolean isNestedScript(String dbCommand) {
-      return dbCommand.startsWith(MSSQL_NESTING_TOKEN);
-    }
-  }
-
-  public static NestedScriptParser getDbCommandParser(String dbName) {
-    return getDbCommandParser(dbName, null);
-  }
-
-  public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) {
-    return getDbCommandParser(dbName, null, null, null, null, metaDbName);
-  }
-
-  public static NestedScriptParser getDbCommandParser(String dbName,
-      String dbOpts, String msUsername, String msPassword,
-      HiveConf hiveConf, String metaDbType) {
-    if (dbName.equalsIgnoreCase(DB_DERBY)) {
-      return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf);
-    } else if (dbName.equalsIgnoreCase(DB_HIVE)) {
-      return new HiveCommandParser(dbOpts, msUsername, msPassword, hiveConf, metaDbType);
-    } else if (dbName.equalsIgnoreCase(DB_MSSQL)) {
-      return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf);
-    } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
-      return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf);
-    } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
-      return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf);
-    } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
-      return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf);
-    } else {
-      throw new IllegalArgumentException("Unknown dbType " + dbName);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
index f312e46..4ff4beb 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
@@ -38,8 +38,10 @@ import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
 import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
 import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -80,6 +82,7 @@ public class HiveSchemaTool {
   private final String dbType;
   private final String metaDbType;
   private final IMetaStoreSchemaInfo metaStoreSchemaInfo;
+  private boolean needsQuotedIdentifier;
 
   static final private Logger LOG = LoggerFactory.getLogger(HiveSchemaTool.class.getName());
 
@@ -95,6 +98,7 @@ public class HiveSchemaTool {
     this.hiveConf = hiveConf;
     this.dbType = dbType;
     this.metaDbType = metaDbType;
+    this.needsQuotedIdentifier = getDbCommandParser(dbType).needsQuotedIdentifier();
     this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf, hiveHome, dbType);
   }
 
@@ -167,42 +171,11 @@ public class HiveSchemaTool {
    * @throws MetaException
    */
   public void showInfo() throws HiveMetaException {
-    Connection metastoreConn = getConnectionToMetastore(true);
     String hiveVersion = metaStoreSchemaInfo.getHiveSchemaVersion();
-    String dbVersion = getMetaStoreSchemaVersion(metastoreConn);
+    String dbVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(true));
     System.out.println("Hive distribution version:\t " + hiveVersion);
     System.out.println("Metastore schema version:\t " + dbVersion);
     assertCompatibleVersion(hiveVersion, dbVersion);
-
-  }
-
-  private String getMetaStoreSchemaVersion(Connection metastoreConn)
-      throws HiveMetaException {
-    return getMetaStoreSchemaVersion(metastoreConn, false);
-  }
-
-  // read schema version from metastore
-  private String getMetaStoreSchemaVersion(Connection metastoreConn,
-      boolean checkDuplicatedVersion) throws HiveMetaException {
-    String versionQuery;
-    if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
-      versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t";
-    } else {
-      versionQuery = "select t.SCHEMA_VERSION from VERSION t";
-    }
-    try(Statement stmt = metastoreConn.createStatement();
-        ResultSet res = stmt.executeQuery(versionQuery)) {
-      if (!res.next()) {
-        throw new HiveMetaException("Could not find version info in metastore VERSION table.");
-      }
-      String currentSchemaVersion = res.getString(1);
-      if (checkDuplicatedVersion && res.next()) {
-        throw new HiveMetaException("Multiple versions were found in metastore.");
-      }
-      return currentSchemaVersion;
-    } catch (SQLException e) {
-      throw new HiveMetaException("Failed to get schema version, Cause:" + e.getMessage());
-    }
   }
 
   boolean validateLocations(Connection conn, URI[] defaultServers) throws HiveMetaException {
@@ -226,7 +199,7 @@ public class HiveSchemaTool {
     String dbLoc;
     boolean isValid = true;
     int numOfInvalid = 0;
-    if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+    if (needsQuotedIdentifier) {
       dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt";
     } else {
       dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt";
@@ -255,13 +228,13 @@ public class HiveSchemaTool {
     String tabLoc, tabIDRange;
     boolean isValid = true;
     int numOfInvalid = 0;
-    if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+    if (needsQuotedIdentifier) {
       tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" ";
     } else {
       tabIDRange = "select max(TBL_ID), min(TBL_ID) from TBLS";
     }
 
-    if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+    if (needsQuotedIdentifier) {
       tabLoc = "select tbl.\"TBL_ID\", tbl.\"TBL_NAME\", sd.\"LOCATION\", dbt.\"DB_ID\", dbt.\"NAME\" from \"TBLS\" tbl inner join " +
     "\"SDS\" sd on tbl.\"SD_ID\" = sd.\"SD_ID\" and tbl.\"TBL_TYPE\" != '" + TableType.VIRTUAL_VIEW +
     "' and tbl.\"TBL_ID\" >= ? and tbl.\"TBL_ID\"<= ? " + "inner join \"DBS\" dbt on tbl.\"DB_ID\" = dbt.\"DB_ID\" ";
@@ -315,13 +288,13 @@ public class HiveSchemaTool {
     String partLoc, partIDRange;
     boolean isValid = true;
     int numOfInvalid = 0;
-    if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+    if (needsQuotedIdentifier) {
       partIDRange = "select max(\"PART_ID\"), min(\"PART_ID\") from \"PARTITIONS\" ";
     } else {
       partIDRange = "select max(PART_ID), min(PART_ID) from PARTITIONS";
     }
 
-    if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+    if (needsQuotedIdentifier) {
       partLoc = "select pt.\"PART_ID\", pt.\"PART_NAME\", sd.\"LOCATION\", tbl.\"TBL_ID\", tbl.\"TBL_NAME\",dbt.\"DB_ID\", dbt.\"NAME\" from \"PARTITIONS\" pt "
            + "inner join \"SDS\" sd on pt.\"SD_ID\" = sd.\"SD_ID\" and pt.\"PART_ID\" >= ? and pt.\"PART_ID\"<= ? "
            + " inner join \"TBLS\" tbl on pt.\"TBL_ID\" = tbl.\"TBL_ID\" inner join "
@@ -376,13 +349,13 @@ public class HiveSchemaTool {
     String skewedColLoc, skewedColIDRange;
     boolean isValid = true;
     int numOfInvalid = 0;
-    if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+    if (needsQuotedIdentifier) {
       skewedColIDRange = "select max(\"STRING_LIST_ID_KID\"), min(\"STRING_LIST_ID_KID\") from \"SKEWED_COL_VALUE_LOC_MAP\" ";
     } else {
       skewedColIDRange = "select max(STRING_LIST_ID_KID), min(STRING_LIST_ID_KID) from SKEWED_COL_VALUE_LOC_MAP";
     }
 
-    if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+    if (needsQuotedIdentifier) {
       skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\", db.\"NAME\", db.\"DB_ID\" from \"TBLS\" t, \"SDS\" s, \"DBS\" db, \"SKEWED_COL_VALUE_LOC_MAP\" sk "
            + "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and t.\"DB_ID\" = db.\"DB_ID\" and sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? ";
     } else {
@@ -497,11 +470,9 @@ public class HiveSchemaTool {
     if (dryRun) {
       return;
     }
-    String newSchemaVersion = getMetaStoreSchemaVersion(
-        getConnectionToMetastore(false));
+    String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
     // verify that the new version is added to schema
     assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
-
   }
 
   private void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion)
@@ -517,8 +488,8 @@ public class HiveSchemaTool {
    * @throws MetaException
    */
   public void doUpgrade() throws HiveMetaException {
-    String fromVersion = getMetaStoreSchemaVersion(
-        getConnectionToMetastore(false));
+    String fromVersion =
+      metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
     if (fromVersion == null || fromVersion.isEmpty()) {
       throw new HiveMetaException("Schema version not stored in the metastore. " +
           "Metastore schema is too old or corrupt. Try specifying the version manually");
@@ -526,6 +497,10 @@ public class HiveSchemaTool {
     doUpgrade(fromVersion);
   }
 
+  private MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) {
+    return new MetaStoreConnectionInfo(userName, passWord, url, driver, printInfo, hiveConf,
+        dbType);
+  }
   /**
    * Perform metastore schema upgrade
    *
@@ -682,10 +657,10 @@ public class HiveSchemaTool {
       for (String seqName : seqNameToTable.keySet()) {
         String tableName = seqNameToTable.get(seqName).getLeft();
         String tableKey = seqNameToTable.get(seqName).getRight();
-        String seqQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ?
+        String seqQuery = needsQuotedIdentifier ?
             ("select t.\"NEXT_VAL\" from \"SEQUENCE_TABLE\" t WHERE t.\"SEQUENCE_NAME\"='org.apache.hadoop.hive.metastore.model." + seqName + "'")
             : ("select t.NEXT_VAL from SEQUENCE_TABLE t WHERE t.SEQUENCE_NAME='org.apache.hadoop.hive.metastore.model." + seqName + "'");
-        String maxIdQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ?
+        String maxIdQuery = needsQuotedIdentifier ?
             ("select max(\"" + tableKey + "\") from \"" + tableName + "\"")
             : ("select max(" + tableKey + ") from " + tableName);
 
@@ -715,7 +690,7 @@ public class HiveSchemaTool {
   boolean validateSchemaVersions(Connection conn) throws HiveMetaException {
     System.out.println("Validating schema version");
     try {
-      String newSchemaVersion = getMetaStoreSchemaVersion(conn, true);
+      String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
       assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
     } catch (HiveMetaException hme) {
       if (hme.getMessage().contains("Metastore schema version is not compatible")
@@ -743,7 +718,7 @@ public class HiveSchemaTool {
 
     System.out.println("Validating metastore schema tables");
     try {
-      version = getMetaStoreSchemaVersion(hmsConn);
+      version = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
     } catch (HiveMetaException he) {
       System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
       System.out.println("Failed in schema version validation.");
@@ -781,8 +756,8 @@ public class HiveSchemaTool {
     // parse the schema file to determine the tables that are expected to exist
     // we are using oracle schema because it is simpler to parse, no quotes or backticks etc
     String baseDir    = new File(metaStoreSchemaInfo.getMetaStoreScriptDir()).getParent();
-    String schemaFile = baseDir  + "/" + dbType + "/hive-schema-" + version + "." + dbType + ".sql";
-
+    String schemaFile = new File(metaStoreSchemaInfo.getMetaStoreScriptDir(),
+        metaStoreSchemaInfo.generateInitFileName(version)).getPath();
     try {
       LOG.debug("Parsing schema script " + schemaFile);
       subScripts.addAll(findCreateTable(schemaFile, schemaTables));
@@ -896,7 +871,7 @@ public class HiveSchemaTool {
     boolean isValid = true;
     try {
       Statement stmt = conn.createStatement();
-      String tblQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ?
+      String tblQuery = needsQuotedIdentifier ?
           ("select t.* from \"TBLS\" t WHERE t.\"SD_ID\" IS NULL and (t.\"TBL_TYPE\"='" + TableType.EXTERNAL_TABLE + "' or t.\"TBL_TYPE\"='" + TableType.MANAGED_TABLE + "')")
           : ("select t.* from TBLS t WHERE t.SD_ID IS NULL and (t.TBL_TYPE='" + TableType.EXTERNAL_TABLE + "' or t.TBL_TYPE='" + TableType.MANAGED_TABLE + "')");
 

http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java b/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
index 716bce7..d9b8f42 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
@@ -18,6 +18,7 @@
 package org.apache.hive.beeline;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
index d1658f7..45172c4 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
@@ -36,11 +36,11 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaException;
 import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo;
 import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.PostgresCommandParser;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
-import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser;
 
 public class TestSchemaTool extends TestCase {
   private HiveSchemaTool schemaTool;

http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
index d662743..296d04b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
@@ -18,7 +18,10 @@
 package org.apache.hadoop.hive.metastore;
 
 import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
+import java.sql.Connection;
 import java.util.List;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
 
 /**
  * Defines the method which must be implemented to be used using schema tool to support metastore
@@ -78,6 +81,18 @@ public interface IMetaStoreSchemaInfo {
   String getHiveSchemaVersion();
 
   /**
+   * Get the schema version from the backend database. This version is used by SchemaTool to to
+   * compare the version returned by getHiveSchemaVersion and determine the upgrade order and
+   * scripts needed to upgrade the metastore schema
+   * 
+   * @param metastoreDbConnectionInfo Connection information needed to connect to the backend
+   *          database
+   * @return
+   * @throws HiveMetaException when unable to fetch the schema version
+   */
+  String getMetaStoreSchemaVersion(
+      HiveSchemaHelper.MetaStoreConnectionInfo metastoreDbConnectionInfo) throws HiveMetaException;
+  /**
    * A dbVersion is compatible with hive version if it is greater or equal to the hive version. This
    * is result of the db schema upgrade design principles followed in hive project. The state where
    * db schema version is ahead of hive software version is often seen when a 'rolling upgrade' or

http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
index 6bddb8e..d6d83e0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
@@ -22,10 +22,16 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
 import org.apache.hive.common.util.HiveVersionInfo;
 
 import com.google.common.collect.ImmutableMap;
@@ -198,4 +204,31 @@ public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo {
     return true;
   }
 
+  @Override
+  public String getMetaStoreSchemaVersion(MetaStoreConnectionInfo connectionInfo)
+      throws HiveMetaException {
+    String versionQuery;
+    boolean needsQuotedIdentifier =
+        HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType()).needsQuotedIdentifier();
+    if (needsQuotedIdentifier) {
+      versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t";
+    } else {
+      versionQuery = "select t.SCHEMA_VERSION from VERSION t";
+    }
+    try (Connection metastoreDbConnection =
+        HiveSchemaHelper.getConnectionToMetastore(connectionInfo)) {
+      Statement stmt = metastoreDbConnection.createStatement();
+      ResultSet res = stmt.executeQuery(versionQuery);
+      if (!res.next()) {
+        throw new HiveMetaException("Could not find version info in metastore VERSION table.");
+      }
+      String currentSchemaVersion = res.getString(1);
+      if (res.next()) {
+        throw new HiveMetaException("Multiple versions were found in metastore.");
+      }
+      return currentSchemaVersion;
+    } catch (SQLException e) {
+      throw new HiveMetaException("Failed to get schema version, Cause:" + e.getMessage());
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
new file mode 100644
index 0000000..0127bdd
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
@@ -0,0 +1,640 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.IllegalFormatException;
+import java.util.List;
+
+public class HiveSchemaHelper {
+  public static final String DB_DERBY = "derby";
+  public static final String DB_HIVE = "hive";
+  public static final String DB_MSSQL = "mssql";
+  public static final String DB_MYSQL = "mysql";
+  public static final String DB_POSTGRACE = "postgres";
+  public static final String DB_ORACLE = "oracle";
+
+  /***
+   * Get JDBC connection to metastore db
+   *
+   * @param userName metastore connection username
+   * @param password metastore connection password
+   * @param printInfo print connection parameters
+   * @param hiveConf hive config object
+   * @return metastore connection object
+   * @throws org.apache.hadoop.hive.metastore.api.MetaException
+   */
+  public static Connection getConnectionToMetastore(String userName,
+      String password, String url, String driver, boolean printInfo,
+      HiveConf hiveConf)
+      throws HiveMetaException {
+    try {
+      url = url == null ? getValidConfVar(
+        HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf) : url;
+      driver = driver == null ? getValidConfVar(
+        HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver;
+      if (printInfo) {
+        System.out.println("Metastore connection URL:\t " + url);
+        System.out.println("Metastore Connection Driver :\t " + driver);
+        System.out.println("Metastore connection User:\t " + userName);
+      }
+      if ((userName == null) || userName.isEmpty()) {
+        throw new HiveMetaException("UserName empty ");
+      }
+
+      // load required JDBC driver
+      Class.forName(driver);
+
+      // Connect using the JDBC URL and user/pass from conf
+      return DriverManager.getConnection(url, userName, password);
+    } catch (IOException e) {
+      throw new HiveMetaException("Failed to get schema version.", e);
+    } catch (SQLException e) {
+      throw new HiveMetaException("Failed to get schema version.", e);
+    } catch (ClassNotFoundException e) {
+      throw new HiveMetaException("Failed to load driver", e);
+    }
+  }
+
+  public static Connection getConnectionToMetastore(MetaStoreConnectionInfo info) throws HiveMetaException {
+    return getConnectionToMetastore(info.getUsername(), info.getPassword(), info.getUrl(),
+        info.getDriver(), info.getPrintInfo(), info.getHiveConf());
+  }
+
+  public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf)
+      throws IOException {
+    String confVarStr = hiveConf.get(confVar.varname);
+    if (confVarStr == null || confVarStr.isEmpty()) {
+      throw new IOException("Empty " + confVar.varname);
+    }
+    return confVarStr.trim();
+  }
+
+  public interface NestedScriptParser {
+
+    public enum CommandType {
+      PARTIAL_STATEMENT,
+      TERMINATED_STATEMENT,
+      COMMENT
+    }
+
+    static final String DEFAULT_DELIMITER = ";";
+    static final String DEFAULT_QUOTE = "\"";
+
+    /**
+     * Find the type of given command
+     *
+     * @param dbCommand
+     * @return
+     */
+    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
+
+    /**
+     * Parse the DB specific nesting format and extract the inner script name if any
+     *
+     * @param dbCommand command from parent script
+     * @return
+     * @throws IllegalFormatException
+     */
+    public String getScriptName(String dbCommand) throws IllegalArgumentException;
+
+    /**
+     * Find if the given command is a nested script execution
+     *
+     * @param dbCommand
+     * @return
+     */
+    public boolean isNestedScript(String dbCommand);
+
+    /**
+     * Find if the given command should not be passed to DB
+     *
+     * @param dbCommand
+     * @return
+     */
+    public boolean isNonExecCommand(String dbCommand);
+
+    /**
+     * Get the SQL statement delimiter
+     *
+     * @return
+     */
+    public String getDelimiter();
+
+    /**
+     * Get the SQL indentifier quotation character
+     *
+     * @return
+     */
+    public String getQuoteCharacter();
+
+    /**
+     * Clear any client specific tags
+     *
+     * @return
+     */
+    public String cleanseCommand(String dbCommand);
+
+    /**
+     * Does the DB required table/column names quoted
+     *
+     * @return
+     */
+    public boolean needsQuotedIdentifier();
+
+    /**
+     * Flatten the nested upgrade script into a buffer
+     *
+     * @param scriptDir  upgrade script directory
+     * @param scriptFile upgrade script file
+     * @return string of sql commands
+     */
+    public String buildCommand(String scriptDir, String scriptFile)
+        throws IllegalFormatException, IOException;
+
+    /**
+     * Flatten the nested upgrade script into a buffer
+     *
+     * @param scriptDir  upgrade script directory
+     * @param scriptFile upgrade script file
+     * @param fixQuotes whether to replace quote characters
+     * @return string of sql commands
+     */
+    public String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes)
+        throws IllegalFormatException, IOException;
+  }
+
+  /***
+   * Base implementation of NestedScriptParser
+   * abstractCommandParser.
+   *
+   */
+  private static abstract class AbstractCommandParser implements NestedScriptParser {
+    private List<String> dbOpts;
+    private String msUsername;
+    private String msPassword;
+    private HiveConf hiveConf;
+
+    public AbstractCommandParser(String dbOpts, String msUsername, String msPassword,
+        HiveConf hiveConf) {
+      setDbOpts(dbOpts);
+      this.msUsername = msUsername;
+      this.msPassword = msPassword;
+      this.hiveConf = hiveConf;
+    }
+
+    @Override
+    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+      if (dbCommand == null || dbCommand.isEmpty()) {
+        throw new IllegalArgumentException("invalid command line " + dbCommand);
+      }
+      dbCommand = dbCommand.trim();
+      if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) {
+        return false;
+      } else {
+        return true;
+      }
+    }
+
+    @Override
+    public boolean isNonExecCommand(String dbCommand) {
+      return (dbCommand.startsWith("--") || dbCommand.startsWith("#"));
+    }
+
+    @Override
+    public String getDelimiter() {
+      return DEFAULT_DELIMITER;
+    }
+
+    @Override
+    public String getQuoteCharacter() {
+      return DEFAULT_QUOTE;
+    }
+
+
+    @Override
+    public String cleanseCommand(String dbCommand) {
+      // strip off the delimiter
+      if (dbCommand.endsWith(getDelimiter())) {
+        dbCommand = dbCommand.substring(0,
+            dbCommand.length() - getDelimiter().length());
+      }
+      return dbCommand;
+    }
+
+    @Override
+    public boolean needsQuotedIdentifier() {
+      return false;
+    }
+
+    @Override
+    public String buildCommand(
+      String scriptDir, String scriptFile) throws IllegalFormatException, IOException {
+      return buildCommand(scriptDir, scriptFile, false);
+    }
+
+    @Override
+    public String buildCommand(
+      String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException {
+      BufferedReader bfReader =
+          new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile));
+      String currLine;
+      StringBuilder sb = new StringBuilder();
+      String currentCommand = null;
+      while ((currLine = bfReader.readLine()) != null) {
+        currLine = currLine.trim();
+
+        if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) {
+          currLine = currLine.replace("\\\"", getQuoteCharacter());
+        }
+
+        if (currLine.isEmpty()) {
+          continue; // skip empty lines
+        }
+
+        if (currentCommand == null) {
+          currentCommand = currLine;
+        } else {
+          currentCommand = currentCommand + " " + currLine;
+        }
+        if (isPartialCommand(currLine)) {
+          // if its a partial line, continue collecting the pieces
+          continue;
+        }
+
+        // if this is a valid executable command then add it to the buffer
+        if (!isNonExecCommand(currentCommand)) {
+          currentCommand = cleanseCommand(currentCommand);
+          if (isNestedScript(currentCommand)) {
+            // if this is a nested sql script then flatten it
+            String currScript = getScriptName(currentCommand);
+            sb.append(buildCommand(scriptDir, currScript));
+          } else {
+            // Now we have a complete statement, process it
+            // write the line to buffer
+            sb.append(currentCommand);
+            sb.append(System.getProperty("line.separator"));
+          }
+        }
+        currentCommand = null;
+      }
+      bfReader.close();
+      return sb.toString();
+    }
+
+    private void setDbOpts(String dbOpts) {
+      if (dbOpts != null) {
+        this.dbOpts = Lists.newArrayList(dbOpts.split(","));
+      } else {
+        this.dbOpts = Lists.newArrayList();
+      }
+    }
+
+    protected List<String> getDbOpts() {
+      return dbOpts;
+    }
+
+    protected String getMsUsername() {
+      return msUsername;
+    }
+
+    protected String getMsPassword() {
+      return msPassword;
+    }
+
+    protected HiveConf getHiveConf() {
+      return hiveConf;
+    }
+  }
+
+  // Derby commandline parser
+  public static class DerbyCommandParser extends AbstractCommandParser {
+    private static final String DERBY_NESTING_TOKEN = "RUN";
+
+    public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
+        HiveConf hiveConf) {
+      super(dbOpts, msUsername, msPassword, hiveConf);
+    }
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+
+      if (!isNestedScript(dbCommand)) {
+        throw new IllegalArgumentException("Not a script format " + dbCommand);
+      }
+      String[] tokens = dbCommand.split(" ");
+      if (tokens.length != 2) {
+        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+      }
+      return tokens[1].replace(";", "").replaceAll("'", "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      // Derby script format is RUN '<file>'
+     return dbCommand.startsWith(DERBY_NESTING_TOKEN);
+    }
+  }
+
+  // Derby commandline parser
+  public static class HiveCommandParser extends AbstractCommandParser {
+    private static String HIVE_NESTING_TOKEN = "SOURCE";
+    private final NestedScriptParser nestedDbCommandParser;
+
+    public HiveCommandParser(String dbOpts, String msUsername, String msPassword,
+        HiveConf hiveConf, String metaDbType) {
+      super(dbOpts, msUsername, msPassword, hiveConf);
+      nestedDbCommandParser = getDbCommandParser(metaDbType);
+    }
+
+    @Override
+    public String getQuoteCharacter() {
+      return nestedDbCommandParser.getQuoteCharacter();
+    }
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+
+      if (!isNestedScript(dbCommand)) {
+        throw new IllegalArgumentException("Not a script format " + dbCommand);
+      }
+      String[] tokens = dbCommand.split(" ");
+      if (tokens.length != 2) {
+        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+      }
+      return tokens[1].replace(";", "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+     return dbCommand.startsWith(HIVE_NESTING_TOKEN);
+    }
+  }
+
+  // MySQL parser
+  public static class MySqlCommandParser extends AbstractCommandParser {
+    private static final String MYSQL_NESTING_TOKEN = "SOURCE";
+    private static final String DELIMITER_TOKEN = "DELIMITER";
+    private String delimiter = DEFAULT_DELIMITER;
+
+    public MySqlCommandParser(String dbOpts, String msUsername, String msPassword,
+        HiveConf hiveConf) {
+      super(dbOpts, msUsername, msPassword, hiveConf);
+    }
+
+    @Override
+    public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+      boolean isPartial = super.isPartialCommand(dbCommand);
+      // if this is a delimiter directive, reset our delimiter
+      if (dbCommand.startsWith(DELIMITER_TOKEN)) {
+        String[] tokens = dbCommand.split(" ");
+        if (tokens.length != 2) {
+          throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+        }
+        delimiter = tokens[1];
+      }
+      return isPartial;
+    }
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+      String[] tokens = dbCommand.split(" ");
+      if (tokens.length != 2) {
+        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+      }
+      // remove ending ';'
+      return tokens[1].replace(";", "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
+    }
+
+    @Override
+    public String getDelimiter() {
+      return delimiter;
+    }
+
+    @Override
+    public String getQuoteCharacter() {
+      return "`";
+    }
+
+    @Override
+    public boolean isNonExecCommand(String dbCommand) {
+      return super.isNonExecCommand(dbCommand) ||
+          (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) ||
+          dbCommand.startsWith(DELIMITER_TOKEN);
+    }
+
+    @Override
+    public String cleanseCommand(String dbCommand) {
+      return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
+    }
+
+  }
+
+  // Postgres specific parser
+  public static class PostgresCommandParser extends AbstractCommandParser {
+    private static final String POSTGRES_NESTING_TOKEN = "\\i";
+    @VisibleForTesting
+    public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
+    @VisibleForTesting
+    public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
+
+    public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
+        HiveConf hiveConf) {
+      super(dbOpts, msUsername, msPassword, hiveConf);
+    }
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+      String[] tokens = dbCommand.split(" ");
+      if (tokens.length != 2) {
+        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+      }
+      // remove ending ';'
+      return tokens[1].replace(";", "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
+    }
+
+    @Override
+    public boolean needsQuotedIdentifier() {
+      return true;
+    }
+
+    @Override
+    public boolean isNonExecCommand(String dbCommand) {
+      // Skip "standard_conforming_strings" command which is read-only in older
+      // Postgres versions like 8.1
+      // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html
+      if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) {
+        if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) {
+          return true;
+        }
+      }
+      return super.isNonExecCommand(dbCommand);
+    }
+  }
+
+  //Oracle specific parser
+  public static class OracleCommandParser extends AbstractCommandParser {
+    private static final String ORACLE_NESTING_TOKEN = "@";
+
+    public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
+        HiveConf hiveConf) {
+      super(dbOpts, msUsername, msPassword, hiveConf);
+    }
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+      if (!isNestedScript(dbCommand)) {
+        throw new IllegalArgumentException("Not a nested script format " + dbCommand);
+      }
+      // remove ending ';' and starting '@'
+      return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
+    }
+  }
+
+  //MSSQL specific parser
+  public static class MSSQLCommandParser extends AbstractCommandParser {
+    private static final String MSSQL_NESTING_TOKEN = ":r";
+
+    public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
+        HiveConf hiveConf) {
+      super(dbOpts, msUsername, msPassword, hiveConf);
+    }
+
+    @Override
+    public String getScriptName(String dbCommand) throws IllegalArgumentException {
+      String[] tokens = dbCommand.split(" ");
+      if (tokens.length != 2) {
+        throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+      }
+      return tokens[1];
+    }
+
+    @Override
+    public boolean isNestedScript(String dbCommand) {
+      return dbCommand.startsWith(MSSQL_NESTING_TOKEN);
+    }
+  }
+
+  public static NestedScriptParser getDbCommandParser(String dbName) {
+    return getDbCommandParser(dbName, null);
+  }
+
+  public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) {
+    return getDbCommandParser(dbName, null, null, null, null, metaDbName);
+  }
+
+  public static NestedScriptParser getDbCommandParser(String dbName,
+      String dbOpts, String msUsername, String msPassword,
+      HiveConf hiveConf, String metaDbType) {
+    if (dbName.equalsIgnoreCase(DB_DERBY)) {
+      return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+    } else if (dbName.equalsIgnoreCase(DB_HIVE)) {
+      return new HiveCommandParser(dbOpts, msUsername, msPassword, hiveConf, metaDbType);
+    } else if (dbName.equalsIgnoreCase(DB_MSSQL)) {
+      return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+    } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
+      return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+    } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
+      return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+    } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
+      return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+    } else {
+      throw new IllegalArgumentException("Unknown dbType " + dbName);
+    }
+  }
+
+  public static class MetaStoreConnectionInfo {
+    private final String userName;
+    private final String password;
+    private final String url;
+    private final String driver;
+    private final boolean printInfo;
+    private final HiveConf hiveConf;
+    private final String dbType;
+
+    public MetaStoreConnectionInfo(String userName, String password, String url, String driver,
+        boolean printInfo, HiveConf hiveConf, String dbType) {
+      super();
+      this.userName = userName;
+      this.password = password;
+      this.url = url;
+      this.driver = driver;
+      this.printInfo = printInfo;
+      this.hiveConf = hiveConf;
+      this.dbType = dbType;
+    }
+
+    public String getPassword() {
+      return password;
+    }
+
+    public String getUrl() {
+      return url;
+    }
+
+    public String getDriver() {
+      return driver;
+    }
+
+    public boolean isPrintInfo() {
+      return printInfo;
+    }
+
+    public HiveConf getHiveConf() {
+      return hiveConf;
+    }
+
+    public String getUsername() {
+      return userName;
+    }
+
+    public boolean getPrintInfo() {
+      return printInfo;
+    }
+
+    public String getDbType() {
+      return dbType;
+    }
+  }
+}


Mime
View raw message