hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1525046 - in /hive/trunk/beeline/src: java/org/apache/hive/beeline/ test/org/apache/hive/beeline/src/test/
Date Fri, 20 Sep 2013 16:07:17 GMT
Author: brock
Date: Fri Sep 20 16:07:16 2013
New Revision: 1525046

URL: http://svn.apache.org/r1525046
Log:
HIVE-4568 - Beeline needs to support resolving variables (Xuefu Zhang reviewed by Thejas M
Nair)

Modified:
    hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java
    hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.properties
    hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
    hive/trunk/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java
    hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java

Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1525046&r1=1525045&r2=1525046&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java Fri Sep 20 16:07:16 2013
@@ -96,7 +96,7 @@ import jline.SimpleCompletor;
 public class BeeLine {
   private static final ResourceBundle resourceBundle =
       ResourceBundle.getBundle(BeeLine.class.getName());
-  private BeeLineSignalHandler signalHandler = null;
+  private final BeeLineSignalHandler signalHandler = null;
   private static final String separator = System.getProperty("line.separator");
   private boolean exit = false;
   private final DatabaseConnections connections = new DatabaseConnections();
@@ -125,6 +125,8 @@ public class BeeLine {
   private static final int ERRNO_ARGS = 1;
   private static final int ERRNO_OTHER = 2;
 
+  private static final String HIVE_VAR_PREFIX = "--hivevar";
+
   private final Map<Object, Object> formats = map(new Object[] {
       "vertical", new VerticalOutputFormat(this),
       "table", new TableOutputFormat(this),
@@ -504,6 +506,16 @@ public class BeeLine {
         return false;
       }
 
+      // Parse hive variables
+      if (args[i].equals(HIVE_VAR_PREFIX)) {
+        String[] parts = split(args[++i], "=");
+        if (parts.length != 2) {
+          return false;
+        }
+        getOpts().getHiveVariables().put(parts[0], parts[1]);
+        continue;
+      }
+
       // -- arguments are treated as properties
       if (args[i].startsWith("--")) {
         String[] parts = split(args[i].substring(2), "=");

Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.properties
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.properties?rev=1525046&r1=1525045&r2=1525046&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.properties (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.properties Fri Sep 20 16:07:16
2013
@@ -143,6 +143,10 @@ cmd-usage: Usage: java org.apache.hive.c
 \  -d <driver class>               the driver class to use\n \
 \  -e <query>                      query that should be executed\n \
 \  -f <file>                       script file that should be executed\n \
+\  --hivevar name=value            hive variable name and value\n \
+\                                  This is Hive specific settings in which variables\n \
+\                                  can be set at session level and referenced in Hive\n \
+\                                  commands or queries.\n \
 \  --color=[true/false]            control whether color is used for display\n \
 \  --showHeader=[true/false]       show column names in query results\n \
 \  --headerInterval=ROWS;          the interval between which heades are displayed\n \

Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java?rev=1525046&r1=1525045&r2=1525046&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java Fri Sep 20 16:07:16
2013
@@ -31,8 +31,10 @@ import java.io.OutputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.TreeSet;
 
@@ -80,6 +82,8 @@ class BeeLineOpts implements Completor {
 
   private String scriptFile = null;
 
+  private Map<String, String> hiveVariables = new HashMap<String, String>();
+
   public BeeLineOpts(BeeLine beeLine, Properties props) {
     this.beeLine = beeLine;
     if (terminal.getTerminalWidth() > 0) {
@@ -421,4 +425,13 @@ class BeeLineOpts implements Completor {
   public File getPropertiesFile() {
     return rcFile;
   }
+
+  public Map<String, String> getHiveVariables() {
+    return hiveVariables;
+  }
+
+  public void setHiveVariables(Map<String, String> hiveVariables) {
+    this.hiveVariables = hiveVariables;
+  }
+
 }

Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java?rev=1525046&r1=1525045&r2=1525046&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/DatabaseConnection.java Fri Sep 20
16:07:16 2013
@@ -28,8 +28,10 @@ import java.sql.DatabaseMetaData;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -52,9 +54,37 @@ class DatabaseConnection {
       String username, String password) throws SQLException {
     this.beeLine = beeLine;
     this.driver = driver;
-    this.url = url;
     this.username = username;
     this.password = password;
+    this.url = appendHiveVariables(beeLine, url);
+  }
+
+/**
+ * Append hive variables specified on the command line to the connection url
+ * (after #). They will be set later on the session on the server side.
+ */
+  private static String appendHiveVariables(BeeLine beeLine, String url) {
+    StringBuilder sb = new StringBuilder( url );
+    Map<String, String> hiveVars = beeLine.getOpts().getHiveVariables();
+    if (hiveVars.size() > 0) {
+      if (url.indexOf("#") == -1) {
+        sb.append("#");
+      } else {
+        sb.append("&");
+      }
+      Set<Map.Entry<String, String>> vars = hiveVars.entrySet();
+      Iterator<Map.Entry<String, String>> it = vars.iterator();
+      while (it.hasNext()) {
+        Map.Entry<String, String> var = it.next();
+        sb.append(var.getKey());
+        sb.append("=");
+        sb.append(var.getValue());
+        if (it.hasNext()) {
+          sb.append("&");
+        }
+      }
+    }
+    return sb.toString();
   }
 
 

Modified: hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java?rev=1525046&r1=1525045&r2=1525046&view=diff
==============================================================================
--- hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
(original)
+++ hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
Fri Sep 20 16:07:16 2013
@@ -23,6 +23,8 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.PrintStream;
 import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hive.beeline.BeeLine;
@@ -38,14 +40,19 @@ import org.junit.Test;
  */
 //public class TestBeeLineWithArgs extends TestCase {
 public class TestBeeLineWithArgs {
-
   // Default location of HiveServer2
-  final static String BASE_JDBC_URL = BeeLine.BEELINE_DEFAULT_JDBC_URL + "localhost:10000";
-  //set JDBC_URL to something else in test case, if it needs to be customized
-  String JDBC_URL = BASE_JDBC_URL;
+  final private static String JDBC_URL = BeeLine.BEELINE_DEFAULT_JDBC_URL + "localhost:10000";
 
   private static HiveServer2 hiveServer2;
 
+  private List<String> getBaseArgs(String jdbcUrl) {
+    List<String> argList = new ArrayList<String>(8);
+    argList.add("-d");
+    argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER);
+    argList.add("-u");
+    argList.add(jdbcUrl);
+    return argList;
+  }
   /**
    * Start up a local Hive Server 2 for these tests
    */
@@ -83,13 +90,13 @@ public class TestBeeLineWithArgs {
    * @throws Any exception while executing
    * @return The stderr and stdout from running the script
    */
-  private String testCommandLineScript(String scriptFileName) throws Throwable {
-    String[] args = {"-d", BeeLine.BEELINE_DEFAULT_JDBC_DRIVER, "-u", JDBC_URL, "-f", scriptFileName};
+  private String testCommandLineScript(List<String> argList) throws Throwable {
     BeeLine beeLine = new BeeLine();
     ByteArrayOutputStream os = new ByteArrayOutputStream();
     PrintStream beelineOutputStream = new PrintStream(os);
     beeLine.setOutputStream(beelineOutputStream);
     beeLine.setErrorStream(beelineOutputStream);
+    String[] args = argList.toArray(new String[argList.size()]);
     beeLine.begin(args, null);
     String output = os.toString("UTF8");
 
@@ -106,7 +113,8 @@ public class TestBeeLineWithArgs {
    * @param shouldMatch true if the pattern should be found, false if it should not
    * @throws Exception on command execution error
    */
-  private void testScriptFile(String testName, String scriptText, String expectedPattern,
boolean shouldMatch) throws Throwable {
+  private void testScriptFile(String testName, String scriptText, String expectedPattern,
+      boolean shouldMatch, List<String> argList) throws Throwable {
 
     long startTime = System.currentTimeMillis();
     System.out.println(">>> STARTED " + testName);
@@ -118,9 +126,12 @@ public class TestBeeLineWithArgs {
     os.print(scriptText);
     os.close();
 
+    argList.add("-f");
+    argList.add(scriptFile.getAbsolutePath());
+
     if(shouldMatch) {
       try {
-        String output = testCommandLineScript(scriptFile.getAbsolutePath());
+        String output = testCommandLineScript(argList);
         long elapsedTime = (System.currentTimeMillis() - startTime)/1000;
         String time = "(" + elapsedTime + "s)";
         if (output.contains(expectedPattern)) {
@@ -136,7 +147,7 @@ public class TestBeeLineWithArgs {
       }
     } else {
       try {
-        String output = testCommandLineScript(scriptFile.getAbsolutePath());
+        String output = testCommandLineScript(argList);
         long elapsedTime = (System.currentTimeMillis() - startTime)/1000;
         String time = "(" + elapsedTime + "s)";
         if (output.contains(expectedPattern)) {
@@ -166,7 +177,25 @@ public class TestBeeLineWithArgs {
     final String TEST_NAME = "testPositiveScriptFile";
     final String SCRIPT_TEXT = "show databases;\n";
     final String EXPECTED_PATTERN = " default ";
-    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true);
+    List<String> argList = getBaseArgs(JDBC_URL);
+    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+  }
+
+  /**
+   * Test Beeline -hivevar option. User can specify --hivevar name=value on Beeline command
line.
+   * In the script, user should be able to use it in the form of ${name}, which will be substituted
with
+   * the value.
+   * @throws Throwable
+   */
+  @Test
+  public void testBeelineCommandLineHiveVariable() throws Throwable {
+    List<String> argList = getBaseArgs(JDBC_URL);
+    argList.add("--hivevar");
+    argList.add("DUMMY_TBL=dummy");
+    final String TEST_NAME = "testHiveCommandLineHiveVariable";
+    final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n";
+    final String EXPECTED_PATTERN = "dummy";
+    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
   }
 
   /**
@@ -176,10 +205,11 @@ public class TestBeeLineWithArgs {
    */
   @Test
   public void testBreakOnErrorScriptFile() throws Throwable {
+    List<String> argList = getBaseArgs(JDBC_URL);
     final String TEST_NAME = "testBreakOnErrorScriptFile";
     final String SCRIPT_TEXT = "select * from abcdefg01;\nshow databases;\n";
     final String EXPECTED_PATTERN = " default ";
-    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false);
+    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false, argList);
   }
 
   /**
@@ -198,8 +228,12 @@ public class TestBeeLineWithArgs {
     File scriptFile = File.createTempFile("beelinenegative", "temp");
     scriptFile.delete();
 
+    List<String> argList = getBaseArgs(JDBC_URL);
+    argList.add("-f");
+    argList.add(scriptFile.getAbsolutePath());
+
     try {
-        String output = testCommandLineScript(scriptFile.getAbsolutePath());
+        String output = testCommandLineScript(argList);
       long elapsedTime = (System.currentTimeMillis() - startTime)/1000;
       String time = "(" + elapsedTime + "s)";
       if (output.contains(EXPECTED_PATTERN)) {
@@ -243,11 +277,11 @@ public class TestBeeLineWithArgs {
 
   @Test
   public void testHiveVarSubstitution() throws Throwable {
-    JDBC_URL = BASE_JDBC_URL + "#D_TBL=dummy_t";
+    List<String> argList = getBaseArgs(JDBC_URL + "#D_TBL=dummy_t");
     final String TEST_NAME = "testHiveVarSubstitution";
     final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\n";
     final String EXPECTED_PATTERN = "dummy_t";
-    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true);
+    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
   }
 
 }



Mime
View raw message