Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 768A3108A9 for ; Tue, 17 Sep 2013 21:59:18 +0000 (UTC) Received: (qmail 15644 invoked by uid 500); 17 Sep 2013 21:58:48 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 15623 invoked by uid 500); 17 Sep 2013 21:58:47 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 15551 invoked by uid 99); 17 Sep 2013 21:58:40 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 17 Sep 2013 21:58:40 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 17 Sep 2013 21:58:38 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 2DA5A23888CD; Tue, 17 Sep 2013 21:58:18 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1524234 - in /hive/trunk/hcatalog: src/test/e2e/templeton/tests/jobsubmission.conf webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Date: Tue, 17 Sep 2013 21:58:18 -0000 To: commits@hive.apache.org From: thejas@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130917215818.2DA5A23888CD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: thejas Date: Tue Sep 17 21:58:17 2013 New Revision: 1524234 URL: http://svn.apache.org/r1524234 Log: HIVE-4444: [HCatalog] WebHCat Hive should support equivalent parameters as Pig (Daniel Dai via Thejas Nair) Modified: hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Modified: hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf?rev=1524234&r1=1524233&r2=1524234&view=diff ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf (original) +++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf Tue Sep 17 21:58:17 2013 @@ -377,7 +377,37 @@ $cfg = }, - + { + # test to use "add jar" command in hive script, the jar must be shipped to + # templeton controller job using "files". Test #9 is a positive test case + # for this case when jar is shipped, #10 is a negative test case + 'num' => 9, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar', 'files=:INPDIR_HDFS:/piggybank.jar',], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_job_exit_value' => 0, + 'check_call_back' => 1, + }, + { + # negative test case for "add jar" when jar is not shipped to templeton + # controller job using "files" + 'num' => 10, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar',], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_job_exit_value' => 1, + 'check_call_back' => 1, + }, ] }, Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1524234&r1=1524233&r2=1524234&view=diff ============================================================================== --- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original) +++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Sep 17 21:58:17 2013 @@ -22,6 +22,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.commons.exec.ExecuteException; @@ -41,24 +42,26 @@ public class HiveDelegator extends Launc public EnqueueBean run(String user, String execute, String srcFile, List defines, + List hiveArgs, String otherFiles, String statusdir, String callback, String completedUrl) throws NotAuthorizedException, BadParam, BusyException, QueueException, ExecuteException, IOException, InterruptedException { runAs = user; - List args = makeArgs(execute, srcFile, defines, statusdir, + List args = makeArgs(execute, srcFile, defines, hiveArgs, otherFiles, statusdir, completedUrl); return enqueueController(user, callback, args); } private List makeArgs(String execute, String srcFile, - List defines, String statusdir, String completedUrl) + List defines, List hiveArgs, String otherFiles, + String statusdir, String completedUrl) throws BadParam, IOException, InterruptedException { ArrayList args = new ArrayList(); try { - args.addAll(makeBasicArgs(execute, srcFile, statusdir, completedUrl)); + args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl)); args.add("--"); args.add(appConf.hivePath()); @@ -77,6 +80,7 @@ public class HiveDelegator extends Launc args.add("--hiveconf"); args.add(prop); } + args.addAll(hiveArgs); if (TempletonUtils.isset(execute)) { args.add("-e"); args.add(execute); @@ -94,8 +98,8 @@ public class HiveDelegator extends Launc return args; } - private List makeBasicArgs(String execute, String srcFile, - String statusdir, String completedUrl) + private List makeBasicArgs(String execute, String srcFile, String otherFiles, + String statusdir, String completedUrl) throws URISyntaxException, FileNotFoundException, IOException, InterruptedException { @@ -106,6 +110,11 @@ public class HiveDelegator extends Launc allFiles.add(TempletonUtils.hadoopFsFilename(srcFile, appConf, runAs)); + if (TempletonUtils.isset(otherFiles)) { + String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs); + allFiles.addAll(Arrays.asList(ofs)); + } + args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles)); args.add("-archives"); Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1524234&r1=1524233&r2=1524234&view=diff ============================================================================== --- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original) +++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Sep 17 21:58:17 2013 @@ -659,12 +659,25 @@ public class Server { /** * Run a Hive job. + * @param execute SQL statement to run, equivalent to "-e" from hive command line + * @param srcFile name of hive script file to run, equivalent to "-f" from hive + * command line + * @param hiveArgs additional command line argument passed to the hive command line. + * Please check https://cwiki.apache.org/Hive/languagemanual-cli.html + * for detailed explanation of command line arguments + * @param otherFiles additional files to be shipped to the launcher, such as the jars + * used in "add jar" statement in hive script + * @param defines shortcut for command line arguments "--define" + * @param statusdir where the stderr/stdout of templeton controller job goes + * @param callback callback url when the hive job finishes */ @POST @Path("hive") @Produces({MediaType.APPLICATION_JSON}) public EnqueueBean hive(@FormParam("execute") String execute, @FormParam("file") String srcFile, + @FormParam("arg") List hiveArgs, + @FormParam("files") String otherFiles, @FormParam("define") List defines, @FormParam("statusdir") String statusdir, @FormParam("callback") String callback) @@ -675,7 +688,7 @@ public class Server { throw new BadParam("Either execute or file parameter required"); HiveDelegator d = new HiveDelegator(appConf); - return d.run(getDoAsUser(), execute, srcFile, defines, + return d.run(getDoAsUser(), execute, srcFile, defines, hiveArgs, otherFiles, statusdir, callback, getCompletedUrl()); }