hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1524234 - in /hive/trunk/hcatalog: src/test/e2e/templeton/tests/jobsubmission.conf webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Date Tue, 17 Sep 2013 21:58:18 GMT
Author: thejas
Date: Tue Sep 17 21:58:17 2013
New Revision: 1524234

URL: http://svn.apache.org/r1524234
Log:
HIVE-4444: [HCatalog] WebHCat Hive should support equivalent parameters as Pig (Daniel Dai
via Thejas Nair)

Modified:
    hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java

Modified: hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf?rev=1524234&r1=1524233&r2=1524234&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf Tue Sep 17 21:58:17
2013
@@ -377,7 +377,37 @@ $cfg = 
 
     },
 
-
+    {
+     # test to use "add jar" command in hive script, the jar must be shipped to
+     # templeton controller job using "files". Test #9 is a positive test case
+     # for this case when jar is shipped, #10 is a negative test case                   
    
+     'num' => 9,
+     'method' => 'POST',
+     'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+     'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar', 'files=:INPDIR_HDFS:/piggybank.jar',],
+     'json_field_substr_match' => { 'id' => '\d+'},
+                                #results
+     'status_code' => 200,
+     'check_job_created' => 1,
+     'check_job_complete' => 'SUCCESS', 
+     'check_job_exit_value' => 0,
+     'check_call_back' => 1,
+    },
+    {
+     # negative test case for "add jar" when jar is not shipped to templeton
+     # controller job using "files"
+     'num' => 10,
+     'method' => 'POST',
+     'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+     'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar',],
+     'json_field_substr_match' => { 'id' => '\d+'},
+                                #results
+     'status_code' => 200,
+     'check_job_created' => 1,
+     'check_job_complete' => 'SUCCESS', 
+     'check_job_exit_value' => 1,
+     'check_call_back' => 1,
+    },
 
    ]
   },

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1524234&r1=1524233&r2=1524234&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
Tue Sep 17 21:58:17 2013
@@ -22,6 +22,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.exec.ExecuteException;
@@ -41,24 +42,26 @@ public class HiveDelegator extends Launc
 
   public EnqueueBean run(String user,
                String execute, String srcFile, List<String> defines,
+               List<String> hiveArgs, String otherFiles,
                String statusdir, String callback, String completedUrl)
     throws NotAuthorizedException, BadParam, BusyException, QueueException,
     ExecuteException, IOException, InterruptedException
   {
     runAs = user;
-    List<String> args = makeArgs(execute, srcFile, defines, statusdir,
+    List<String> args = makeArgs(execute, srcFile, defines, hiveArgs, otherFiles, statusdir,
                    completedUrl);
 
     return enqueueController(user, callback, args);
   }
 
   private List<String> makeArgs(String execute, String srcFile,
-      List<String> defines, String statusdir, String completedUrl)
+             List<String> defines, List<String> hiveArgs, String otherFiles,
+             String statusdir, String completedUrl)
     throws BadParam, IOException, InterruptedException
   {
     ArrayList<String> args = new ArrayList<String>();
     try {
-      args.addAll(makeBasicArgs(execute, srcFile, statusdir, completedUrl));
+      args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl));
       args.add("--");
       args.add(appConf.hivePath());
 
@@ -77,6 +80,7 @@ public class HiveDelegator extends Launc
         args.add("--hiveconf");
         args.add(prop);
       }
+      args.addAll(hiveArgs);
       if (TempletonUtils.isset(execute)) {
         args.add("-e");
         args.add(execute);
@@ -94,8 +98,8 @@ public class HiveDelegator extends Launc
     return args;
   }
 
-  private List<String> makeBasicArgs(String execute, String srcFile,
-                     String statusdir, String completedUrl)
+  private List<String> makeBasicArgs(String execute, String srcFile, String otherFiles,
+                                         String statusdir, String completedUrl)
     throws URISyntaxException, FileNotFoundException, IOException,
     InterruptedException
   {
@@ -106,6 +110,11 @@ public class HiveDelegator extends Launc
       allFiles.add(TempletonUtils.hadoopFsFilename(srcFile, appConf,
           runAs));
 
+    if (TempletonUtils.isset(otherFiles)) {
+      String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs);
+      allFiles.addAll(Arrays.asList(ofs));
+    }
+
     args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles));
 
     args.add("-archives");

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1524234&r1=1524233&r2=1524234&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Tue Sep 17 21:58:17 2013
@@ -659,12 +659,25 @@ public class Server {
 
   /**
    * Run a Hive job.
+   * @param execute    SQL statement to run, equivalent to "-e" from hive command line
+   * @param srcFile    name of hive script file to run, equivalent to "-f" from hive
+   *                   command line
+   * @param hiveArgs   additional command line argument passed to the hive command line.

+   *                   Please check https://cwiki.apache.org/Hive/languagemanual-cli.html
+   *                   for detailed explanation of command line arguments
+   * @param otherFiles additional files to be shipped to the launcher, such as the jars
+   *                   used in "add jar" statement in hive script
+   * @param defines    shortcut for command line arguments "--define"
+   * @param statusdir  where the stderr/stdout of templeton controller job goes
+   * @param callback   callback url when the hive job finishes
    */
   @POST
   @Path("hive")
   @Produces({MediaType.APPLICATION_JSON})
   public EnqueueBean hive(@FormParam("execute") String execute,
               @FormParam("file") String srcFile,
+              @FormParam("arg") List<String> hiveArgs,
+              @FormParam("files") String otherFiles,
               @FormParam("define") List<String> defines,
               @FormParam("statusdir") String statusdir,
               @FormParam("callback") String callback)
@@ -675,7 +688,7 @@ public class Server {
       throw new BadParam("Either execute or file parameter required");
 
     HiveDelegator d = new HiveDelegator(appConf);
-    return d.run(getDoAsUser(), execute, srcFile, defines,
+    return d.run(getDoAsUser(), execute, srcFile, defines, hiveArgs, otherFiles,
       statusdir, callback, getCompletedUrl());
   }
 



Mime
View raw message