hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x..@apache.org
Subject [10/33] hive git commit: HIVE-6791: Support variable substition for Beeline shell command (Ferdinand Xu, reviewed by Xuefu Zhang)
Date Mon, 28 Sep 2015 01:10:30 GMT
HIVE-6791: Support variable substition for Beeline shell command (Ferdinand Xu, reviewed by Xuefu Zhang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/102b23b1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/102b23b1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/102b23b1

Branch: refs/heads/master
Commit: 102b23b16bf26cbf439009b4b95542490a082710
Parents: 00e0d55
Author: Ferdinand Xu <cheng.a.xu@intel.com>
Authored: Sun Jun 28 20:42:54 2015 -0400
Committer: Ferdinand Xu <cheng.a.xu@intel.com>
Committed: Sun Jun 28 20:42:54 2015 -0400

----------------------------------------------------------------------
 .../java/org/apache/hive/beeline/BeeLine.java   |  75 +---
 .../java/org/apache/hive/beeline/Commands.java  | 430 ++++++++++++++-----
 .../apache/hive/beeline/cli/TestHiveCli.java    |  75 +++-
 .../org/apache/hadoop/hive/cli/CliDriver.java   |  24 +-
 .../hadoop/hive/conf/HiveVariableSource.java    |  24 ++
 .../hadoop/hive/conf/VariableSubstitution.java  |  70 +++
 .../hive/conf/TestVariableSubstitution.java     |  63 +++
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  10 +-
 .../ql/parse/ColumnStatsSemanticAnalyzer.java   |  14 +-
 .../hive/ql/parse/VariableSubstitution.java     |  60 ---
 .../ql/processors/AddResourceProcessor.java     |  11 +-
 .../hive/ql/processors/CompileProcessor.java    |  11 +-
 .../ql/processors/DeleteResourceProcessor.java  |  11 +-
 .../hadoop/hive/ql/processors/DfsProcessor.java |  11 +-
 .../hadoop/hive/ql/processors/SetProcessor.java |  32 +-
 .../service/cli/operation/SQLOperation.java     |  10 +-
 16 files changed, 651 insertions(+), 280 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/beeline/src/java/org/apache/hive/beeline/BeeLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index 66fe322..1d468eb 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -87,6 +87,7 @@ import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hive.beeline.cli.CliOptionsProcessor;
 import org.apache.hive.jdbc.Utils;
@@ -830,7 +831,7 @@ public class BeeLine implements Closeable {
       }
 
       if (getOpts().getScriptFile() != null) {
-        return executeFile(getOpts().getScriptFile(), false);
+        return executeFile(getOpts().getScriptFile());
       }
       try {
         info(getApplicationTitle());
@@ -849,7 +850,7 @@ public class BeeLine implements Closeable {
     if (initFile != null) {
       info("Running init script " + initFile);
       try {
-        return executeFile(initFile, false);
+        return executeFile(initFile);
       } finally {
         exit = false;
       }
@@ -884,7 +885,7 @@ public class BeeLine implements Closeable {
     return ERRNO_OK;
   }
 
-  private int executeFile(String fileName, boolean isSourceCMD) {
+  private int executeFile(String fileName) {
     FileInputStream initStream = null;
     try {
       initStream = new FileInputStream(fileName);
@@ -894,31 +895,8 @@ public class BeeLine implements Closeable {
       return ERRNO_OTHER;
     } finally {
       IOUtils.closeStream(initStream);
-      if(!isSourceCMD) {
-        consoleReader = null;
-        output("");   // dummy new line
-      }
-    }
-  }
-
-  private boolean isSourceCMD(String cmd) {
-    if (cmd == null || cmd.isEmpty())
-      return false;
-    String[] tokens = tokenizeCmd(cmd);
-    return tokens[0].equalsIgnoreCase("!source");
-  }
-
-  private boolean sourceFile(String cmd) {
-    String[] tokens = tokenizeCmd(cmd);
-    String cmd_1 = getFirstCmd(cmd, tokens[0].length());
-    File sourceFile = new File(cmd_1);
-    if (!sourceFile.isFile()) {
-      return false;
-    } else {
-      boolean ret = (executeFile(cmd_1, true) == ERRNO_OK);
-      // For source command, we should not exit even when meeting some empty line.
-      setExit(false);
-      return ret;
+      consoleReader = null;
+      output("");   // dummy new line
     }
   }
 
@@ -933,10 +911,6 @@ public class BeeLine implements Closeable {
 
         // trim line
         line = (line == null) ? null : line.trim();
-        if (!isBeeLine) {
-          line = cliToBeelineCmd(line);
-        }
-
         if (!dispatch(line) && exitOnError) {
           return ERRNO_OTHER;
         }
@@ -1043,31 +1017,6 @@ public class BeeLine implements Closeable {
   }
 
   /**
-   * Extract and clean up the first command in the input.
-   */
-  private String getFirstCmd(String cmd, int length) {
-    return cmd.substring(length).trim();
-  }
-
-  private String cliToBeelineCmd(String cmd) {
-    if (cmd == null)
-      return null;
-    String[] tokens = tokenizeCmd(cmd);
-    if (cmd.equalsIgnoreCase("quit") || cmd.equalsIgnoreCase("exit")
-	|| cmd.equalsIgnoreCase("quit;") || cmd.equals("exit;")) {
-      return null;
-    } else if (tokens[0].equalsIgnoreCase("source")) {
-      return COMMAND_PREFIX + cmd;
-    } else if (cmd.startsWith("!")) {
-      String shell_cmd = cmd.substring(1);
-      return "!sh " + shell_cmd;
-    } else { // local mode
-      // command like dfs
-      return cmd;
-    }
-  }
-
-  /**
    * Dispatch the specified line to the appropriate {@link CommandHandler}.
    *
    * @param line
@@ -1089,10 +1038,6 @@ public class BeeLine implements Closeable {
       return true;
     }
 
-    if(isSourceCMD(line)){
-      return sourceFile(line);
-    }
-
     line = line.trim();
 
     // save it to the current script, if any
@@ -2121,4 +2066,12 @@ public class BeeLine implements Closeable {
   protected Reflector getReflector() {
     return reflector;
   }
+
+  public boolean isBeeLine() {
+    return isBeeLine;
+  }
+
+  public void setBeeLine(boolean isBeeLine) {
+    this.isBeeLine = isBeeLine;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/beeline/src/java/org/apache/hive/beeline/Commands.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java
index aaf6aec..d490273 100644
--- a/beeline/src/java/org/apache/hive/beeline/Commands.java
+++ b/beeline/src/java/org/apache/hive/beeline/Commands.java
@@ -22,6 +22,10 @@
  */
 package org.apache.hive.beeline;
 
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.SystemVariables;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.io.IOUtils;
 
 import java.io.BufferedReader;
@@ -32,7 +36,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.reflect.Method;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.sql.CallableStatement;
@@ -44,9 +47,11 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.SQLWarning;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.TreeSet;
@@ -219,9 +224,8 @@ public class Commands {
 
 
   public boolean exportedkeys(String line) throws Exception {
-    return metadata("getExportedKeys", new String[] {
-        beeLine.getConnection().getCatalog(), null,
-        arg1(line, "table name"),});
+    return metadata("getExportedKeys",
+        new String[] { beeLine.getConnection().getCatalog(), null, arg1(line, "table name"), });
   }
 
 
@@ -709,10 +713,303 @@ public class Commands {
     return execute(line, false, false);
   }
 
+  /**
+   * This method is used for retrieving the latest configuration from hive server2.
+   * It uses the set command processor.
+   *
+   * @return
+   */
+  private Map<String, String> getHiveVariables() {
+    Map<String, String> result = new HashMap<>();
+    BufferedRows rows = getConfInternal();
+    while (rows.hasNext()) {
+      Rows.Row row = (Rows.Row) rows.next();
+      if (!row.isMeta) {
+        result.put(row.values[0], row.values[1]);
+      }
+    }
+    return result;
+  }
+
+  private HiveConf getHiveConf() {
+    HiveConf conf = new HiveConf();
+    BufferedRows rows = getConfInternal();
+    while (rows.hasNext()) {
+      addConf((Rows.Row) rows.next(), conf);
+    }
+    return conf;
+  }
+
+  private BufferedRows getConfInternal() {
+    Statement stmnt = null;
+    BufferedRows rows = null;
+    try {
+      stmnt = beeLine.createStatement();
+      boolean hasResults = stmnt.execute("set");
+      if (hasResults) {
+        ResultSet rs = stmnt.getResultSet();
+        rows = new BufferedRows(beeLine, rs);
+      }
+    } catch (SQLException e) {
+      beeLine.error(e);
+      if (stmnt != null) {
+        try {
+          stmnt.close();
+        } catch (SQLException e1) {
+          beeLine.error(e1);
+        }
+      }
+    }
+    return rows;
+  }
+
+  private void addConf(Rows.Row r, HiveConf hiveConf) {
+    if (r.isMeta) {
+      return;
+    }
+    if (r.values == null || r.values[0] == null || r.values[0].isEmpty()) {
+      return;
+    }
+    String val = r.values[0];
+    if (r.values[0].startsWith(SystemVariables.SYSTEM_PREFIX) || r.values[0]
+        .startsWith(SystemVariables.ENV_PREFIX)) {
+      return;
+    } else {
+      String[] kv = val.split("=", 2);
+      hiveConf.set(kv[0], kv[1]);
+    }
+  }
+
+  /**
+   * Extract and clean up the first command in the input.
+   */
+  private String getFirstCmd(String cmd, int length) {
+    return cmd.substring(length).trim();
+  }
+
+  private String[] tokenizeCmd(String cmd) {
+    return cmd.split("\\s+");
+  }
+
+  private boolean isSourceCMD(String cmd) {
+    if (cmd == null || cmd.isEmpty())
+      return false;
+    String[] tokens = tokenizeCmd(cmd);
+    return tokens[0].equalsIgnoreCase("!source");
+  }
+
+  private boolean sourceFile(String cmd) {
+    String[] tokens = tokenizeCmd(cmd);
+    String cmd_1 = getFirstCmd(cmd, tokens[0].length());
+
+    cmd_1 = substituteVariables(getHiveConf(), cmd_1);
+    File sourceFile = new File(cmd_1);
+    if (!sourceFile.isFile()) {
+      return false;
+    } else {
+      boolean ret;
+      try {
+        ret = sourceFileInternal(sourceFile);
+      } catch (IOException e) {
+        beeLine.error(e);
+        return false;
+      }
+      return ret;
+    }
+  }
+
+  private boolean sourceFileInternal(File sourceFile) throws IOException {
+    BufferedReader reader = null;
+    try {
+      reader = new BufferedReader(new FileReader(sourceFile));
+      String extra = reader.readLine();
+      String lines = null;
+      while (extra != null) {
+        if (beeLine.isComment(extra)) {
+          continue;
+        }
+        if (lines == null) {
+          lines = extra;
+        } else {
+          lines += "\n" + extra;
+        }
+        extra = reader.readLine();
+      }
+      String[] cmds = lines.split(";");
+      for (String c : cmds) {
+        if (!executeInternal(c, false)) {
+          return false;
+        }
+      }
+    } finally {
+      if (reader != null) {
+        reader.close();
+      }
+    }
+    return true;
+  }
+
+  private String cliToBeelineCmd(String cmd) {
+    if (cmd == null)
+      return null;
+    String[] tokens = tokenizeCmd(cmd);
+    if (tokens[0].equalsIgnoreCase("source")) {
+      return BeeLine.COMMAND_PREFIX + cmd;
+    } else if (cmd.startsWith("!")) {
+      String shell_cmd = cmd.substring(1);
+      return "!sh " + shell_cmd;
+    } else { // local mode
+      // command like dfs
+      return cmd;
+    }
+  }
+
+  // Return false only occurred error when execution the sql and the sql should follow the rules
+  // of beeline.
+  private boolean executeInternal(String sql, boolean call) {
+    if (sql == null || sql.length() == 0) {
+      return true;
+    }
+
+    if (beeLine.isComment(sql)) {
+      //skip this and rest cmds in the line
+      return true;
+    }
+
+    if (sql.startsWith(BeeLine.COMMAND_PREFIX)) {
+      sql = sql.substring(1);
+    }
+
+    String prefix = call ? "call" : "sql";
+
+    if (sql.startsWith(prefix)) {
+      sql = sql.substring(prefix.length());
+    }
+
+    // batch statements?
+    if (beeLine.getBatch() != null) {
+      beeLine.getBatch().add(sql);
+      return true;
+    }
+
+    try {
+      Statement stmnt = null;
+      boolean hasResults;
+      Thread logThread = null;
+
+      try {
+        long start = System.currentTimeMillis();
+
+        if (call) {
+          stmnt = beeLine.getDatabaseConnection().getConnection().prepareCall(sql);
+          hasResults = ((CallableStatement) stmnt).execute();
+        } else {
+          stmnt = beeLine.createStatement();
+          if (beeLine.getOpts().isSilent()) {
+            hasResults = stmnt.execute(sql);
+          } else {
+            logThread = new Thread(createLogRunnable(stmnt));
+            logThread.setDaemon(true);
+            logThread.start();
+            hasResults = stmnt.execute(sql);
+            logThread.interrupt();
+            logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
+          }
+        }
+
+        beeLine.showWarnings();
+
+        if (hasResults) {
+          do {
+            ResultSet rs = stmnt.getResultSet();
+            try {
+              int count = beeLine.print(rs);
+              long end = System.currentTimeMillis();
+
+              beeLine.info(
+                  beeLine.loc("rows-selected", count) + " " + beeLine.locElapsedTime(end - start));
+            } finally {
+              if (logThread != null) {
+                logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
+                showRemainingLogsIfAny(stmnt);
+                logThread = null;
+              }
+              rs.close();
+            }
+          } while (BeeLine.getMoreResults(stmnt));
+        } else {
+          int count = stmnt.getUpdateCount();
+          long end = System.currentTimeMillis();
+          beeLine.info(
+              beeLine.loc("rows-affected", count) + " " + beeLine.locElapsedTime(end - start));
+        }
+      } finally {
+        if (logThread != null) {
+          if (!logThread.isInterrupted()) {
+            logThread.interrupt();
+          }
+          logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
+          showRemainingLogsIfAny(stmnt);
+        }
+        if (stmnt != null) {
+          stmnt.close();
+        }
+      }
+    } catch (Exception e) {
+      return beeLine.error(e);
+    }
+    beeLine.showWarnings();
+    return true;
+  }
+
+  public String handleMultiLineCmd(String line) throws IOException {
+    //When using -e, console reader is not initialized and command is a single line
+    while (beeLine.getConsoleReader() != null && !(line.trim().endsWith(";")) && beeLine.getOpts()
+        .isAllowMultiLineCommand()) {
+
+      if (!beeLine.getOpts().isSilent()) {
+        StringBuilder prompt = new StringBuilder(beeLine.getPrompt());
+        for (int i = 0; i < prompt.length() - 1; i++) {
+          if (prompt.charAt(i) != '>') {
+            prompt.setCharAt(i, i % 2 == 0 ? '.' : ' ');
+          }
+        }
+      }
+
+      String extra;
+      if (beeLine.getOpts().isSilent() && beeLine.getOpts().getScriptFile() != null) {
+        extra = beeLine.getConsoleReader().readLine(null, jline.console.ConsoleReader.NULL_MASK);
+      } else {
+        extra = beeLine.getConsoleReader().readLine(beeLine.getPrompt());
+      }
+
+      if (extra == null) { //it happens when using -f and the line of cmds does not end with ;
+        break;
+      }
+      if (!beeLine.isComment(extra)) {
+        line += "\n" + extra;
+      }
+    }
+    return line;
+  }
+
   public boolean sql(String line, boolean entireLineAsCommand) {
     return execute(line, false, entireLineAsCommand);
   }
 
+  private String substituteVariables(HiveConf conf, String line) {
+    if (!beeLine.isBeeLine()) {
+      // Substitution is only supported in non-beeline mode.
+      return new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return getHiveVariables();
+        }
+      }).substitute(conf, line);
+    }
+    return line;
+  }
+
   public boolean sh(String line) {
     if (line == null || line.length() == 0) {
       return false;
@@ -723,9 +1020,7 @@ public class Commands {
     }
 
     line = line.substring("sh".length()).trim();
-
-    // Support variable substitution. HIVE-6791.
-    // line = new VariableSubstitution().substitute(new HiveConf(BeeLine.class), line.trim());
+    line = substituteVariables(getHiveConf(), line.trim());
 
     try {
       ShellCmdExecutor executor = new ShellCmdExecutor(line, beeLine.getOutputStream(),
@@ -738,7 +1033,6 @@ public class Commands {
       return true;
     } catch (Exception e) {
       beeLine.error("Exception raised from Shell command " + e);
-      beeLine.error(e);
       return false;
     }
   }
@@ -760,33 +1054,7 @@ public class Commands {
 
     // use multiple lines for statements not terminated by ";"
     try {
-      //When using -e, console reader is not initialized and command is a single line
-      while (beeLine.getConsoleReader() != null && !(line.trim().endsWith(";"))
-        && beeLine.getOpts().isAllowMultiLineCommand()) {
-
-        if (!beeLine.getOpts().isSilent()) {
-          StringBuilder prompt = new StringBuilder(beeLine.getPrompt());
-          for (int i = 0; i < prompt.length() - 1; i++) {
-            if (prompt.charAt(i) != '>') {
-              prompt.setCharAt(i, i % 2 == 0 ? '.' : ' ');
-            }
-          }
-        }
-
-        String extra = null;
-        if (beeLine.getOpts().isSilent() && beeLine.getOpts().getScriptFile() != null) {
-          extra = beeLine.getConsoleReader().readLine(null, jline.console.ConsoleReader.NULL_MASK);
-        } else {
-          extra = beeLine.getConsoleReader().readLine(beeLine.getPrompt());
-        }
-
-        if (extra == null) { //it happens when using -f and the line of cmds does not end with ;
-          break;
-        }
-        if (!beeLine.isComment(extra)) {
-          line += "\n" + extra;
-        }
-      }
+      line = handleMultiLineCmd(line);
     } catch (Exception e) {
       beeLine.handleException(e);
     }
@@ -806,93 +1074,23 @@ public class Commands {
     for (int i = 0; i < cmds.length; i++) {
       String sql = cmds[i].trim();
       if (sql.length() != 0) {
-        if (beeLine.isComment(sql)) {
-          //skip this and rest cmds in the line
-          break;
-        }
-        if (sql.startsWith(BeeLine.COMMAND_PREFIX)) {
-          sql = sql.substring(1);
-        }
-
-        String prefix = call ? "call" : "sql";
-
-        if (sql.startsWith(prefix)) {
-          sql = sql.substring(prefix.length());
+        if (!beeLine.isBeeLine()) {
+          sql = cliToBeelineCmd(sql);
+          if (sql.equalsIgnoreCase("quit") || sql.equalsIgnoreCase("exit")) {
+            beeLine.setExit(true);
+            return true;
+          }
         }
 
-        // batch statements?
-        if (beeLine.getBatch() != null) {
-          beeLine.getBatch().add(sql);
+        // is source CMD
+        if (isSourceCMD(sql)) {
+          sourceFile(sql);
           continue;
         }
 
-        try {
-          Statement stmnt = null;
-          boolean hasResults;
-          Thread logThread = null;
-
-          try {
-            long start = System.currentTimeMillis();
-
-            if (call) {
-              stmnt = beeLine.getDatabaseConnection().getConnection().prepareCall(sql);
-              hasResults = ((CallableStatement) stmnt).execute();
-            } else {
-              stmnt = beeLine.createStatement();
-              if (beeLine.getOpts().isSilent()) {
-                hasResults = stmnt.execute(sql);
-              } else {
-                logThread = new Thread(createLogRunnable(stmnt));
-                logThread.setDaemon(true);
-                logThread.start();
-                hasResults = stmnt.execute(sql);
-                logThread.interrupt();
-                logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
-              }
-            }
-
-            beeLine.showWarnings();
-
-            if (hasResults) {
-              do {
-                ResultSet rs = stmnt.getResultSet();
-                try {
-                  int count = beeLine.print(rs);
-                  long end = System.currentTimeMillis();
-
-                  beeLine.info(beeLine.loc("rows-selected", count) + " "
-                      + beeLine.locElapsedTime(end - start));
-                } finally {
-                  if (logThread != null) {
-                    logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
-                    showRemainingLogsIfAny(stmnt);
-                    logThread = null;
-                  }
-                  rs.close();
-                }
-              } while (BeeLine.getMoreResults(stmnt));
-            } else {
-              int count = stmnt.getUpdateCount();
-              long end = System.currentTimeMillis();
-              beeLine.info(beeLine.loc("rows-affected", count)
-                  + " " + beeLine.locElapsedTime(end - start));
-            }
-          } finally {
-            if (logThread != null) {
-              if (!logThread.isInterrupted()) {
-                logThread.interrupt();
-              }
-              logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
-              showRemainingLogsIfAny(stmnt);
-            }
-            if (stmnt != null) {
-              stmnt.close();
-            }
-          }
-        } catch (Exception e) {
-          return beeLine.error(e);
+        if (!executeInternal(sql, call)) {
+          return false;
         }
-        beeLine.showWarnings();
       }
     }
     return true;

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java
index 6cbb030..ff8ab17 100644
--- a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java
+++ b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -42,6 +42,10 @@ public class TestHiveCli {
 
   private final static String SOURCE_CONTEXT =
       "create table if not exists test.testSrcTbl(a string, b string);";
+  private final static String SOURCE_CONTEXT2 =
+      "create table if not exists test.testSrcTbl2(a string);";
+  private final static String SOURCE_CONTEXT3 =
+      "create table if not exists test.testSrcTbl3(a string);";
   final static String CMD =
       "create database if not exists test;\ncreate table if not exists test.testTbl(a string, b "
           + "string);\n";
@@ -60,7 +64,7 @@ public class TestHiveCli {
         inputStream = IOUtils.toInputStream(input);
       }
       ret = cli.runWithArgs(args, inputStream);
-    } catch (IOException e) {
+    } catch (Throwable e) {
       LOG.error("Failed to execute command due to the error: " + e);
     } finally {
       if (retCode != ret) {
@@ -77,48 +81,81 @@ public class TestHiveCli {
     Assert.assertTrue(output.contains(keywords));
   }
 
-  @Test public void testInValidCmd() {
-    verifyCMD("!lss\n", "Failed to execute lss", errS, null, ERRNO_OK);
+  @Test
+  public void testInValidCmd() {
+    verifyCMD("!lss\n", "Unknown command: lss", errS, null, ERRNO_OK);
   }
 
-  @Test public void testHelp() {
+  @Test
+  public void testHelp() {
     verifyCMD(null, "usage: hive", os, new String[] { "-H" }, ERRNO_ARGS);
   }
 
-  @Test public void testInvalidDatabaseOptions() {
-    verifyCMD("\nshow tables\nquit\n", "Database does not exist: invalidDB", errS,
+  @Test
+  public void testInvalidDatabaseOptions() {
+    verifyCMD("\nshow tables;\nquit;\n", "Database does not exist: invalidDB", errS,
         new String[] { "--database", "invalidDB" }, ERRNO_OK);
   }
 
-  @Test public void testDatabaseOptions() {
+  @Test
+  public void testDatabaseOptions() {
     verifyCMD("\nshow tables;\nquit;", "testTbl", os, new String[] { "--database", "test" },
         ERRNO_OK);
   }
 
-  @Test public void testSourceCmd() {
+  @Test
+  public void testSourceCmd() {
     File f = generateTmpFile(SOURCE_CONTEXT);
-    verifyCMD("source " + f.getPath() + "\n" + "desc testSrcTbl\n" + "quit\n", "col_name", os,
+    verifyCMD("source " + f.getPath() + ";" + "desc testSrcTbl;\nquit;\n", "col_name", os,
         new String[] { "--database", "test" }, ERRNO_OK);
+    f.delete();
   }
 
-  @Test public void testSqlFromCmd() {
+  @Test
+  public void testSourceCmd2() {
+    File f = generateTmpFile(SOURCE_CONTEXT3);
+    verifyCMD("source " + f.getPath() + ";" + "desc testSrcTbl3;\nquit;\n", "col_name", os,
+        new String[] { "--database", "test" }, ERRNO_OK);
+    f.delete();
+  }
+
+  @Test
+  public void testSqlFromCmd() {
     verifyCMD(null, "", os, new String[] { "-e", "show databases;" }, ERRNO_OK);
   }
 
-  @Test public void testSqlFromCmdWithDBName() {
+  @Test
+  public void testSqlFromCmdWithDBName() {
     verifyCMD(null, "testTbl", os, new String[] { "-e", "show tables;", "--database", "test" },
         ERRNO_OK);
   }
 
-  @Test public void testInvalidOptions() {
+  @Test
+  public void testInvalidOptions() {
     verifyCMD(null, "The '-e' and '-f' options cannot be specified simultaneously", errS,
         new String[] { "-e", "show tables;", "-f", "path/to/file" }, ERRNO_ARGS);
   }
 
-  @Test public void testInvalidOptions2() {
+  @Test
+  public void testInvalidOptions2() {
     verifyCMD(null, "Unrecognized option: -k", errS, new String[] { "-k" }, ERRNO_ARGS);
   }
 
+  @Test
+  public void testVariables() {
+    verifyCMD("set system:xxx=5;\nset system:yyy=${system:xxx};\nset system:yyy;", "", os, null,
+        ERRNO_OK);
+  }
+
+  @Test
+  public void testVariablesForSource() {
+    File f = generateTmpFile(SOURCE_CONTEXT2);
+    verifyCMD(
+        "set hiveconf:zzz=" + f.getAbsolutePath() + ";\nsource ${hiveconf:zzz};\ndesc testSrcTbl2;",
+        "col_name", os, new String[] { "--database", "test" }, ERRNO_OK);
+    f.delete();
+  }
+
   private void redirectOutputStream() {
     // Setup output stream to redirect output to
     os = new ByteArrayOutputStream();
@@ -152,13 +189,15 @@ public class TestHiveCli {
     return file;
   }
 
-  @Before public void setup() {
+  @Before
+  public void setup() {
     cli = new HiveCli();
     redirectOutputStream();
     initFromFile();
   }
 
-  @After public void tearDown() {
+  @After
+  public void tearDown() {
     tmp.delete();
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
----------------------------------------------------------------------
diff --git a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
index d62fd5c..4b52578 100644
--- a/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
+++ b/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
@@ -61,7 +61,9 @@ import org.apache.hadoop.hive.common.cli.ShellCmdExecutor;
 import org.apache.hadoop.hive.common.io.CachingPrintStream;
 import org.apache.hadoop.hive.common.io.FetchConverter;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
 import org.apache.hadoop.hive.conf.Validator;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.Driver;
@@ -69,7 +71,6 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper;
 import org.apache.hadoop.hive.ql.exec.tez.TezJobExecHelper;
 import org.apache.hadoop.hive.ql.parse.HiveParser;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.processors.CommandProcessor;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -127,7 +128,12 @@ public class CliDriver {
 
     } else if (tokens[0].equalsIgnoreCase("source")) {
       String cmd_1 = getFirstCmd(cmd_trimmed, tokens[0].length());
-      cmd_1 = new VariableSubstitution().substitute(ss.getConf(), cmd_1);
+      cmd_1 = new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return SessionState.get().getHiveVariables();
+        }
+      }).substitute(ss.getConf(), cmd_1);
 
       File sourceFile = new File(cmd_1);
       if (! sourceFile.isFile()){
@@ -145,7 +151,12 @@ public class CliDriver {
     } else if (cmd_trimmed.startsWith("!")) {
 
       String shell_cmd = cmd_trimmed.substring(1);
-      shell_cmd = new VariableSubstitution().substitute(ss.getConf(), shell_cmd);
+      shell_cmd = new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return SessionState.get().getHiveVariables();
+        }
+      }).substitute(ss.getConf(), shell_cmd);
 
       // shell_cmd = "/bin/bash -c \'" + shell_cmd + "\'";
       try {
@@ -671,7 +682,12 @@ public class CliDriver {
 
     // read prompt configuration and substitute variables.
     prompt = conf.getVar(HiveConf.ConfVars.CLIPROMPT);
-    prompt = new VariableSubstitution().substitute(conf, prompt);
+    prompt = new VariableSubstitution(new HiveVariableSource() {
+      @Override
+      public Map<String, String> getHiveVariable() {
+        return SessionState.get().getHiveVariables();
+      }
+    }).substitute(conf, prompt);
     prompt2 = spacesForString(prompt);
 
     SessionState.start(ss);

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/common/src/java/org/apache/hadoop/hive/conf/HiveVariableSource.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveVariableSource.java b/common/src/java/org/apache/hadoop/hive/conf/HiveVariableSource.java
new file mode 100644
index 0000000..86f7a9c
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveVariableSource.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.conf;
+
+import java.util.Map;
+
+public interface HiveVariableSource {
+  Map<String, String> getHiveVariable();
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/common/src/java/org/apache/hadoop/hive/conf/VariableSubstitution.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/VariableSubstitution.java b/common/src/java/org/apache/hadoop/hive/conf/VariableSubstitution.java
new file mode 100644
index 0000000..e1f53ba
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/conf/VariableSubstitution.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.Map;
+
+public class VariableSubstitution extends SystemVariables {
+  private static final Log l4j = LogFactory.getLog(VariableSubstitution.class);
+
+  private HiveVariableSource hiveVariableSource;
+
+  public VariableSubstitution(HiveVariableSource hiveVariableSource) {
+    this.hiveVariableSource = hiveVariableSource;
+  }
+
+  /**
+   * The super method will handle with the case of substitutions for system variables,
+   * hive conf variables and env variables. In this method, it will retrieve the hive
+   * variables using hiveVariableSource.
+   *
+   * @param conf
+   * @param var
+   * @return
+   */
+  @Override
+  protected String getSubstitute(Configuration conf, String var) {
+    String val = super.getSubstitute(conf, var);
+    if (val == null && hiveVariableSource != null) {
+      Map<String, String> vars = hiveVariableSource.getHiveVariable();
+      if (var.startsWith(HIVEVAR_PREFIX)) {
+        val = vars.get(var.substring(HIVEVAR_PREFIX.length()));
+      } else {
+        val = vars.get(var);
+      }
+    }
+    return val;
+  }
+
+  public String substitute(HiveConf conf, String expr) {
+    if (expr == null) {
+      return expr;
+    }
+    if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEVARIABLESUBSTITUTE)) {
+      l4j.debug("Substitution is on: " + expr);
+    } else {
+      return expr;
+    }
+    int depth = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVEVARIABLESUBSTITUTEDEPTH);
+    return substitute(conf, expr, depth);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/common/src/test/org/apache/hadoop/hive/conf/TestVariableSubstitution.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestVariableSubstitution.java b/common/src/test/org/apache/hadoop/hive/conf/TestVariableSubstitution.java
new file mode 100644
index 0000000..faa07a7
--- /dev/null
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestVariableSubstitution.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.conf;
+
+import junit.framework.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestVariableSubstitution {
+  private static class LocalMySource {
+    final Map<String, String> map = new HashMap<>();
+
+    public void put(String k, String v) {
+      map.put(k, v);
+    }
+
+    public String get(String k) {
+      return map.get(k);
+    }
+  }
+
+  private static LocalMySource getMySource() {
+    return localSource.get();
+  }
+
+  private static ThreadLocal<LocalMySource> localSource = new ThreadLocal<LocalMySource>() {
+    @Override protected LocalMySource initialValue() {
+      return new LocalMySource();
+    }
+  };
+
+  @Test public void testVariableSource() throws InterruptedException {
+    final VariableSubstitution variableSubstitution =
+        new VariableSubstitution(new HiveVariableSource() {
+          @Override public Map<String, String> getHiveVariable() {
+            return TestVariableSubstitution.getMySource().map;
+          }
+        });
+
+    String v = variableSubstitution.substitute(new HiveConf(), "${a}");
+    Assert.assertEquals("${a}", v);
+    TestVariableSubstitution.getMySource().put("a", "b");
+    v = variableSubstitution.substitute(new HiveConf(), "${a}");
+    Assert.assertEquals("b", v);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 669e6be..140d6aa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -96,7 +98,6 @@ import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
 import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
 import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
 import org.apache.hadoop.hive.ql.plan.HiveOperation;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -385,7 +386,12 @@ public class Driver implements CommandProcessor {
     SessionState.get().setupQueryCurrentTimestamp();
 
     try {
-      command = new VariableSubstitution().substitute(conf,command);
+      command = new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return SessionState.get().getHiveVariables();
+        }
+      }).substitute(conf, command);
       ctx = new Context(conf);
       ctx.setTryCount(getTryCount());
       ctx.setCmd(command);

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
index a5f0a7f..da2a902 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
@@ -28,12 +28,13 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.serde.serdeConstants;
@@ -216,10 +217,10 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
       //for other usually not used types, just quote the value
       returnVal = "'" + partVal + "'";
     }
-    
+
     return returnVal;
   }
-  
+
   private String getColTypeOf (String partKey) throws SemanticException{
 
     for (FieldSchema fs : tbl.getPartitionKeys()) {
@@ -333,7 +334,12 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
     }
 
     rewrittenQuery = rewrittenQueryBuilder.toString();
-    rewrittenQuery = new VariableSubstitution().substitute(conf, rewrittenQuery);
+    rewrittenQuery = new VariableSubstitution(new HiveVariableSource() {
+      @Override
+      public Map<String, String> getHiveVariable() {
+        return SessionState.get().getHiveVariables();
+      }
+    }).substitute(conf, rewrittenQuery);
     return rewrittenQuery;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java
deleted file mode 100644
index e8b1d96..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.parse;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.conf.SystemVariables;
-
-import java.util.Map;
-
-public class VariableSubstitution extends SystemVariables {
-
-  private static final Log l4j = LogFactory.getLog(VariableSubstitution.class);
-
-  @Override
-  protected String getSubstitute(Configuration conf, String var) {
-    String val = super.getSubstitute(conf, var);
-    if (val == null && SessionState.get() != null) {
-      Map<String,String> vars = SessionState.get().getHiveVariables();
-      if (var.startsWith(HIVEVAR_PREFIX)) {
-        val =  vars.get(var.substring(HIVEVAR_PREFIX.length()));
-      } else {
-        val = vars.get(var);
-      }
-    }
-    return val;
-  }
-
-  public String substitute(HiveConf conf, String expr) {
-    if (expr == null) {
-      return expr;
-    }
-    if (HiveConf.getBoolVar(conf, ConfVars.HIVEVARIABLESUBSTITUTE)) {
-      l4j.debug("Substitution is on: " + expr);
-    } else {
-      return expr;
-    }
-    int depth = HiveConf.getIntVar(conf, ConfVars.HIVEVARIABLESUBSTITUTEDEPTH);
-    return substitute(conf, expr, depth);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
index 0558c53..d2ac993 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
@@ -19,11 +19,13 @@
 package org.apache.hadoop.hive.ql.processors;
 
 import java.util.Arrays;
+import java.util.Map;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -45,7 +47,12 @@ public class AddResourceProcessor implements CommandProcessor {
   @Override
   public CommandProcessorResponse run(String command) {
     SessionState ss = SessionState.get();
-    command = new VariableSubstitution().substitute(ss.getConf(),command);
+    command = new VariableSubstitution(new HiveVariableSource() {
+      @Override
+      public Map<String, String> getHiveVariable() {
+        return SessionState.get().getHiveVariables();
+      }
+    }).substitute(ss.getConf(),command);
     String[] tokens = command.split("\\s+");
     SessionState.ResourceType t;
     if (tokens.length < 2

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java
index 25ce168..7b79f64 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CompileProcessor.java
@@ -24,6 +24,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.Arrays;
+import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -32,9 +33,10 @@ import org.apache.commons.compress.archivers.jar.JarArchiveOutputStream;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -142,7 +144,12 @@ public class CompileProcessor implements CommandProcessor {
   @VisibleForTesting
   void parse(SessionState ss) throws CompileProcessorException {
     if (ss != null){
-      command = new VariableSubstitution().substitute(ss.getConf(), command);
+      command = new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return SessionState.get().getHiveVariables();
+        }
+      }).substitute(ss.getConf(), command);
     }
     if (command == null || command.length() == 0) {
       throw new CompileProcessorException("Command was empty");

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
index 9052c82..736fa9c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
@@ -19,11 +19,13 @@
 package org.apache.hadoop.hive.ql.processors;
 
 import java.util.Arrays;
+import java.util.Map;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -44,7 +46,12 @@ public class DeleteResourceProcessor implements CommandProcessor {
   @Override
   public CommandProcessorResponse run(String command) {
     SessionState ss = SessionState.get();
-    command = new VariableSubstitution().substitute(ss.getConf(),command);
+    command = new VariableSubstitution(new HiveVariableSource() {
+      @Override
+      public Map<String, String> getHiveVariable() {
+        return SessionState.get().getHiveVariables();
+      }
+    }).substitute(ss.getConf(), command);
     String[] tokens = command.split("\\s+");
 
     SessionState.ResourceType t;

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java
index cc0414d..c3d5f81 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/DfsProcessor.java
@@ -20,14 +20,16 @@ package org.apache.hadoop.hive.ql.processors;
 
 import java.io.PrintStream;
 import java.util.Arrays;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Schema;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -65,7 +67,12 @@ public class DfsProcessor implements CommandProcessor {
 
     try {
       SessionState ss = SessionState.get();
-      command = new VariableSubstitution().substitute(ss.getConf(),command);
+      command = new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return SessionState.get().getHiveVariables();
+        }
+      }).substitute(ss.getConf(), command);
 
       String[] tokens = command.split("\\s+");
       CommandProcessorResponse authErrResp =

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
index d271d6d..e5f5f71 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
@@ -30,10 +30,11 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Schema;
 import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.session.SessionState;
 
 /**
@@ -121,17 +122,33 @@ public class SetProcessor implements CommandProcessor {
       return 1;
     } else if (varname.startsWith(SYSTEM_PREFIX)){
       String propName = varname.substring(SYSTEM_PREFIX.length());
-      System.getProperties().setProperty(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue));
+      System.getProperties()
+          .setProperty(propName, new VariableSubstitution(new HiveVariableSource() {
+            @Override
+            public Map<String, String> getHiveVariable() {
+              return SessionState.get().getHiveVariables();
+            }
+          }).substitute(ss.getConf(), varvalue));
     } else if (varname.startsWith(HIVECONF_PREFIX)){
       String propName = varname.substring(HIVECONF_PREFIX.length());
       setConf(varname, propName, varvalue, false);
     } else if (varname.startsWith(HIVEVAR_PREFIX)) {
       String propName = varname.substring(HIVEVAR_PREFIX.length());
-      ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue));
+      ss.getHiveVariables().put(propName, new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return SessionState.get().getHiveVariables();
+        }
+      }).substitute(ss.getConf(), varvalue));
     } else if (varname.startsWith(METACONF_PREFIX)) {
       String propName = varname.substring(METACONF_PREFIX.length());
       Hive hive = Hive.get(ss.getConf());
-      hive.setMetaConf(propName, new VariableSubstitution().substitute(ss.getConf(), varvalue));
+      hive.setMetaConf(propName, new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return SessionState.get().getHiveVariables();
+        }
+      }).substitute(ss.getConf(), varvalue));
     } else {
       setConf(varname, varname, varvalue, true);
       if (varname.equals(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED.toString())) {
@@ -145,7 +162,12 @@ public class SetProcessor implements CommandProcessor {
   private static void setConf(String varname, String key, String varvalue, boolean register)
         throws IllegalArgumentException {
     HiveConf conf = SessionState.get().getConf();
-    String value = new VariableSubstitution().substitute(conf, varvalue);
+    String value = new VariableSubstitution(new HiveVariableSource() {
+      @Override
+      public Map<String, String> getHiveVariable() {
+        return SessionState.get().getHiveVariables();
+      }
+    }).substitute(conf, varvalue);
     if (conf.getBoolVar(HiveConf.ConfVars.HIVECONFVALIDATION)) {
       HiveConf.ConfVars confVars = HiveConf.getConfVars(key);
       if (confVars != null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/102b23b1/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index cc9df76..f4334e4 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -32,6 +32,8 @@ import java.util.concurrent.RejectedExecutionException;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Schema;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
@@ -39,7 +41,6 @@ import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.exec.ExplainTask;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.OperationLog;
 import org.apache.hadoop.hive.ql.session.SessionState;
@@ -105,7 +106,12 @@ public class SQLOperation extends ExecuteStatementOperation {
       // For now, we disable the test attempts.
       driver.setTryCount(Integer.MAX_VALUE);
 
-      String subStatement = new VariableSubstitution().substitute(sqlOperationConf, statement);
+      String subStatement = new VariableSubstitution(new HiveVariableSource() {
+        @Override
+        public Map<String, String> getHiveVariable() {
+          return SessionState.get().getHiveVariables();
+        }
+      }).substitute(sqlOperationConf, statement);
       response = driver.compileAndRespond(subStatement);
       if (0 != response.getResponseCode()) {
         throw toSQLException("Error while compiling statement", response);


Mime
View raw message