hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1590179 - in /hive/trunk/hcatalog: src/test/e2e/templeton/ src/test/e2e/templeton/drivers/ src/test/e2e/templeton/inpdir/ src/test/e2e/templeton/inpdir/sqoop/ src/test/e2e/templeton/tests/ webhcat/svr/src/main/config/ webhcat/svr/src/main/...
Date Fri, 25 Apr 2014 22:42:28 GMT
Author: thejas
Date: Fri Apr 25 22:42:27 2014
New Revision: 1590179

URL: http://svn.apache.org/r1590179
Log:
HIVE-5072 : [WebHCat]Enable directly invoke Sqoop job through Templeton (Shuaishuai Nie reviewed
by Eugene Koifman, Thejas Nair)

Added:
    hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoop/
    hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoop/person.txt
    hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoopcommand.txt
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
Modified:
    hive/trunk/hcatalog/src/test/e2e/templeton/README.txt
    hive/trunk/hcatalog/src/test/e2e/templeton/build.xml
    hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
    hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
    hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/VersionDelegator.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java

Modified: hive/trunk/hcatalog/src/test/e2e/templeton/README.txt
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/README.txt?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/README.txt (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/README.txt Fri Apr 25 22:42:27 2014
@@ -151,6 +151,27 @@ To do this 3 properties in hive-site.xml
     org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener
 4) hive.metastore.execute.setugi set to true
 
+Running Sqoop jobsubmission tests (This test is targeted on Sqoop1. The test will fail if
using Sqoop2) 
+---------------------------------
+ant clean test -Dinpdir.hdfs=<location of inpdir on hdfs> -Ddb.connection.string=<jdbc
connection string>
+    -Ddb.user.name=<DBUserName> -Ddb.password=<DBPassWord> -Dtest.user.name=<user
the tests should run as>
+    -Dharness.webhdfs.url=<webhdfs url upto port num> -Dharness.templeton.url=<templeton
url upto port num>
+    -Dtests.to.run=-t TestSqoop
+
+In order to run Sqoop jobsubmission tests, a RDBMS like MySQL or SQL server should be installed.
Also since
+Sqoop export command require table already exists in the database, a table "PERSON" need
to be created under
+the default database of the RDBMS installed.
+
+Here is the schema of the table writen in MySQL:
+    CREATE TABLE `world`.`person` (
+    `id` INT NOT NULL,
+    `name` VARCHAR(45) NULL,
+    `occupation` VARCHAR(45) NULL,
+    PRIMARY KEY (`id`));
+
+To prevent primary key violation and sqoop import directory conflict, make sure the "PERSON"
table is empty
+and the folder hdfs://hostname:8020/sqoopoutputdir doesn't exist before running the test.
+
 Notes
 -----
 

Modified: hive/trunk/hcatalog/src/test/e2e/templeton/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/build.xml?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/build.xml (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/build.xml Fri Apr 25 22:42:27 2014
@@ -98,6 +98,9 @@
             <env key="TH_WORKING_DIR" value="${test.location}"/>
             <env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
             <env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
+            <env key="DB_CONNECTION_STRING" value="${db.connection.string}"/>
+            <env key="DB_USER_NAME" value="${db.user.name}"/>
+            <env key="DB_PASSWORD" value="${db.password}"/>
             <env key="TH_OUT" value="."/>
             <env key="TH_ROOT" value="."/>
             <env key="FORK_FACTOR_GROUP" value="${fork.factor.group}"/>

Modified: hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Fri Apr 25 22:42:27
2014
@@ -188,6 +188,9 @@ sub globalSetup
 
     $globalHash->{'inpdir_local'} = $ENV{'TH_INPDIR_LOCAL'};
     $globalHash->{'inpdir_hdfs'} = $ENV{'TH_INPDIR_HDFS'};
+    $globalHash->{'db_connection_string'} = $ENV{'DB_CONNECTION_STRING'};
+    $globalHash->{'db_user_name'} = $ENV{'DB_USER_NAME'};
+    $globalHash->{'db_password'} = $ENV{'DB_PASSWORD'};
 
     $globalHash->{'is_secure_mode'} = $ENV{'SECURE_MODE'};
 
@@ -355,6 +358,9 @@ sub replaceParametersInArg
     $arg =~ s/:OUTDIR:/$outdir/g;
     $arg =~ s/:INPDIR_HDFS:/$testCmd->{'inpdir_hdfs'}/g;
     $arg =~ s/:INPDIR_LOCAL:/$testCmd->{'inpdir_local'}/g;
+    $arg =~ s/:DB_CONNECTION_STRING:/$testCmd->{'db_connection_string'}/g;
+    $arg =~ s/:DB_USER_NAME:/$testCmd->{'db_user_name'}/g;
+    $arg =~ s/:DB_PASSWORD:/$testCmd->{'db_password'}/g;
     $arg =~ s/:TNUM:/$testCmd->{'num'}/g;
     return $arg;
   }

Added: hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoop/person.txt
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoop/person.txt?rev=1590179&view=auto
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoop/person.txt (added)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoop/person.txt Fri Apr 25 22:42:27
2014
@@ -0,0 +1,3 @@
+1,Jason,Doctor
+2,David,Engineer
+3,John,Teacher
\ No newline at end of file

Added: hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoopcommand.txt
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoopcommand.txt?rev=1590179&view=auto
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoopcommand.txt (added)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/sqoopcommand.txt Fri Apr 25 22:42:27
2014
@@ -0,0 +1,4 @@
+--table
+person
+--target-dir
+/sqoopoutputdir
\ No newline at end of file

Modified: hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf Fri Apr 25 22:42:27
2014
@@ -218,8 +218,8 @@ $cfg = 
      'status_code' => 400,
 
     },
-	
-	{
+
+    {
                                 #Auto add quote around args
      'ignore' => 'MS9 feature, will reenable later',
      'num' => 9,
@@ -383,7 +383,7 @@ $cfg = 
                                 #test select a,b
      'num' => 7,
      'method' => 'POST',
-     'url' => ':TEMPLETON_URL:/templeton/v1/hive',	
+     'url' => ':TEMPLETON_URL:/templeton/v1/hive',
      'post_options' => ['user.name=:UNAME:','execute=select count(*) from mynums', ],
      'json_field_substr_match' => { 'id' => '\d+'},
                                 #results
@@ -477,7 +477,7 @@ $cfg = 
      'num' => 13,
      'ignore23' => 'Log collector does not work with Hadoop 2',
      'method' => 'POST',
-     'url' => ':TEMPLETON_URL:/templeton/v1/hive',	
+     'url' => ':TEMPLETON_URL:/templeton/v1/hive',
      'post_options' => ['user.name=:UNAME:','execute=select a,b from mynums', 'statusdir=:OUTDIR:/status',
'enablelog=true'],
      'json_field_substr_match' => { 'id' => '\d+'},
                                 #results
@@ -490,6 +490,63 @@ $cfg = 
     },
    ]
   },
+##=============================================================================================================
+  {
+   'name' => 'TestSqoop',
+   'tests' =>
+   [
+    {
+     # Test Sqoop Export
+     'num' => 1,
+     'method' => 'POST',
+     'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:',
+     'post_options' => ['command=export --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME:
--password :DB_PASSWORD: --export-dir :INPDIR_HDFS:/sqoop --table person','statusdir=sqoop.output'
],
+     'json_field_substr_match' => { 'id' => '\d+'},
+                   #results
+     'status_code' => 200,
+     'check_job_created' => 1,
+     'check_job_complete' => 'SUCCESS',
+     'check_job_percent_complete' => 'map 100% reduce 0%',
+     'check_job_exit_value' => 0,
+     'check_call_back' => 1,
+    },
+    {
+     # Test Sqoop Import and option file
+     'num' => 2,
+     'method' => 'POST',
+     'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:',
+     'post_options' => ['files=:INPDIR_HDFS:/sqoopcommand.txt','command=import --connect
:DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --options-file sqoopcommand.txt','statusdir=sqoop.output'
],
+     'json_field_substr_match' => { 'id' => '\d+'},
+                   #results
+     'status_code' => 200,
+     'check_job_created' => 1,
+     'check_job_complete' => 'SUCCESS',
+     'check_job_percent_complete' => 'map 100% reduce 0%',
+     'check_job_exit_value' => 0,
+     'check_call_back' => 1,
+    },
+    {
+     # Test Sqoop using incomplete command
+     'num' => 3,
+     'method' => 'POST',
+     'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:',
+     'post_options' => ['files=:INPDIR_HDFS:/sqoopcommand.txt','statusdir=sqoop.output'
],
+     'json_field_substr_match' => { 'error' => 'Must define Sqoop command or a optionsfile
contains Sqoop command to run Sqoop job.'},
+                   #results
+     'status_code' => 400,
+    },
+    {
+     # Test Sqoop using incorrect command
+     'num' => 4,
+     'method' => 'POST',
+     'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:',
+     'post_options' => ['optionsfile=:INPDIR_HDFS:/sqoopcommand.txt','command=import --connect
:DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --options-file sqoopcommand.txt','statusdir=sqoop.output'
],
+     'json_field_substr_match' => { 'error' => 'Cannot set command and optionsfile
at the same time.'},
+                   #results
+     'status_code' => 400,
+    },
+   ]
+  },
  ]
 },
   ;

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Fri Apr 25 22:42:27
2014
@@ -142,6 +142,12 @@
   </property>
 
   <property>
+    <name>templeton.sqoop.path</name>
+    <value>${env.SQOOP_HOME}/bin/sqoop.cmd</value>
+    <description>The path to the Sqoop executable.</description>
+  </property>
+
+  <property>
     <name>templeton.exec.encoding</name>
     <value>UTF-8</value>
     <description>The encoding of the stdout and stderr data.</description>

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
Fri Apr 25 22:42:27 2014
@@ -114,6 +114,8 @@ public class AppConfig extends Configura
    * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR}
    */
   public static final String HIVE_PROPS_NAME     = "templeton.hive.properties";
+  public static final String SQOOP_ARCHIVE_NAME  = "templeton.sqoop.archive";
+  public static final String SQOOP_PATH_NAME     = "templeton.sqoop.path";
   public static final String LIB_JARS_NAME       = "templeton.libjars";
   public static final String PIG_ARCHIVE_NAME    = "templeton.pig.archive";
   public static final String PIG_PATH_NAME       = "templeton.pig.path";
@@ -249,6 +251,8 @@ public class AppConfig extends Configura
   public String pigArchive()       { return get(PIG_ARCHIVE_NAME); }
   public String hivePath()         { return get(HIVE_PATH_NAME); }
   public String hiveArchive()      { return get(HIVE_ARCHIVE_NAME); }
+  public String sqoopPath()        { return get(SQOOP_PATH_NAME); }
+  public String sqoopArchive()     { return get(SQOOP_ARCHIVE_NAME); }
   public String streamingJar()     { return get(STREAMING_JAR_NAME); }
   public String kerberosSecret()   { return get(KERBEROS_SECRET); }
   public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
Fri Apr 25 22:42:27 2014
@@ -44,7 +44,7 @@ import org.apache.hive.hcatalog.templeto
 public class LauncherDelegator extends TempletonDelegator {
   private static final Log LOG = LogFactory.getLog(LauncherDelegator.class);
   protected String runAs = null;
-  static public enum JobType {JAR, STREAMING, PIG, HIVE}
+  static public enum JobType {JAR, STREAMING, PIG, HIVE, SQOOP}
   private boolean secureMeatastoreAccess = false;
 
   public LauncherDelegator(AppConfig appConf) {

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java
Fri Apr 25 22:42:27 2014
@@ -182,6 +182,8 @@ public class Main {
              FilterMapping.REQUEST);
     root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/hive/*", 
              FilterMapping.REQUEST);
+    root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/sqoop/*",
+             FilterMapping.REQUEST);
     root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/queue/*", 
              FilterMapping.REQUEST);
     root.addFilter(fHolder, "/" + SERVLET_PATH + "/v1/jobs/*",

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Fri Apr 25 22:42:27 2014
@@ -177,7 +177,18 @@ public class Server {
   }
 
   /**
-   * Get version of hive software being run by this WebHCat server
+   * Get version of sqoop software being run by this WebHCat server
+   */
+  @GET
+  @Path("version/sqoop")
+  @Produces(MediaType.APPLICATION_JSON)
+  public Response sqoopVersion()  throws IOException {
+    VersionDelegator d = new VersionDelegator(appConf);
+    return d.getVersion("sqoop");
+  }
+
+  /**
+   * Get version of pig software being run by this WebHCat server
    */
   @GET
   @Path("version/pig")
@@ -762,6 +773,47 @@ public class Server {
       statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog);
   }
 
+   /**
+   * Run a Sqoop job.
+   * @param optionsFile  name of option file which contains Sqoop command to run
+   * @param otherFiles   additional files to be shipped to the launcher, such as option
+                         files which contain part of the Sqoop command
+   * @param statusdir    where the stderr/stdout of templeton controller job goes
+   * @param callback     URL which WebHCat will call when the sqoop job finishes
+   * @param enablelog    whether to collect mapreduce log into statusdir/logs
+   */
+  @POST
+  @Path("sqoop")
+  @Produces({MediaType.APPLICATION_JSON})
+  public EnqueueBean sqoop(@FormParam("command") String command,
+              @FormParam("optionsfile") String optionsFile,
+              @FormParam("files") String otherFiles,
+              @FormParam("statusdir") String statusdir,
+              @FormParam("callback") String callback,
+              @FormParam("enablelog") boolean enablelog)
+    throws NotAuthorizedException, BusyException, BadParam, QueueException,
+    ExecuteException, IOException, InterruptedException {
+    verifyUser();
+    if (command == null && optionsFile == null)
+      throw new BadParam("Must define Sqoop command or a optionsfile contains Sqoop command
to run Sqoop job.");
+    if (command != null && optionsFile != null)
+      throw new BadParam("Cannot set command and optionsfile at the same time.");
+    checkEnableLogPrerequisite(enablelog, statusdir);
+
+    //add all function arguments to a map
+    Map<String, Object> userArgs = new HashMap<String, Object>();
+    userArgs.put("user.name", getDoAsUser());
+    userArgs.put("command", command);
+    userArgs.put("optionsfile", optionsFile);
+    userArgs.put("files", otherFiles);
+    userArgs.put("statusdir", statusdir);
+    userArgs.put("callback", callback);
+    userArgs.put("enablelog", Boolean.toString(enablelog));
+    SqoopDelegator d = new SqoopDelegator(appConf);
+    return d.run(getDoAsUser(), userArgs, command, optionsFile, otherFiles,
+      statusdir, callback, getCompletedUrl(), enablelog);
+  }
+
   /**
    * Run a Hive job.
    * @param execute    SQL statement to run, equivalent to "-e" from hive command line

Added: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java?rev=1590179&view=auto
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
(added)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
Fri Apr 25 22:42:27 2014
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.hcatalog.templeton;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.exec.ExecuteException;
+import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
+import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
+
+/**
+ * Submit a Sqoop job.
+ *
+ * This is the backend of the Sqoop web service.
+ */
+public class SqoopDelegator extends LauncherDelegator {
+
+  public SqoopDelegator(AppConfig appConf) {
+    super(appConf);
+  }
+
+  public EnqueueBean run(String user,
+               Map<String, Object> userArgs, String command, 
+               String optionsFile, String otherFiles, String statusdir, 
+               String callback, String completedUrl, boolean enablelog)
+  throws NotAuthorizedException, BadParam, BusyException, QueueException,
+  ExecuteException, IOException, InterruptedException
+  {
+    runAs = user;
+    List<String> args = makeArgs(command, optionsFile, otherFiles, statusdir,
+                   completedUrl, enablelog);
+
+    return enqueueController(user, userArgs, callback, args);
+  }
+
+  List<String> makeArgs(String command, String optionsFile, String otherFiles,
+            String statusdir, String completedUrl, boolean enablelog)
+    throws BadParam, IOException, InterruptedException
+  {
+    ArrayList<String> args = new ArrayList<String>();
+    try {
+      args.addAll(makeBasicArgs(optionsFile, otherFiles, statusdir, completedUrl, enablelog));
+      args.add("--");
+      TempletonUtils.addCmdForWindows(args);
+      args.add(appConf.sqoopPath());
+      if (TempletonUtils.isset(command)) {
+        String[] temArgs = command.split(" ");
+        for (int i = 0; i < temArgs.length; i++) {
+          args.add(TempletonUtils.quoteForWindows(temArgs[i]));
+
+          // The token file location should be right after the tool argument
+          if (i == 0 && !temArgs[i].startsWith("--")) {
+            args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
+          }
+        }
+      } else if (TempletonUtils.isset(optionsFile)) {
+        args.add("--options-file");
+        args.add(TempletonUtils.hadoopFsPath(optionsFile, appConf, runAs)
+                        .getName());
+      }
+    } catch (FileNotFoundException e) {
+      throw new BadParam(e.getMessage());
+    } catch (URISyntaxException e) {
+      throw new BadParam(e.getMessage());
+    }
+    return args;
+  }
+
+  private List<String> makeBasicArgs(String optionsFile, String otherFiles,
+            String statusdir, String completedUrl, boolean enablelog)
+    throws URISyntaxException, FileNotFoundException, IOException,
+                          InterruptedException
+  {
+    ArrayList<String> args = new ArrayList<String>();
+    ArrayList<String> allFiles = new ArrayList<String>();
+    if (TempletonUtils.isset(optionsFile))
+      allFiles.add(TempletonUtils.hadoopFsFilename(optionsFile, appConf, runAs));
+    if (TempletonUtils.isset(otherFiles)) {
+      String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs);
+      allFiles.addAll(Arrays.asList(ofs));
+    }
+    args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles,
+                enablelog, JobType.SQOOP));
+    if (appConf.sqoopArchive() != null && !appConf.sqoopArchive().equals("")) {
+      args.add("-archives");
+      args.add(appConf.sqoopArchive());
+    }
+    return args;
+  }
+}

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/VersionDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/VersionDelegator.java?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/VersionDelegator.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/VersionDelegator.java
Fri Apr 25 22:42:27 2014
@@ -41,6 +41,8 @@ public class VersionDelegator extends Te
       return getHadoopVersion();
     } else if (module.toLowerCase().equals("hive")) {
       return getHiveVersion();
+    } else if (module.toLowerCase().equals("sqoop")) {
+      return getSqoopVersion();
     } else if (module.toLowerCase().equals("pig")) {
       return getPigVersion();
     } else {
@@ -65,6 +67,11 @@ public class VersionDelegator extends Te
         .build();
   }
 
+  private Response getSqoopVersion() {
+    return SimpleWebException.buildMessage(HttpStatus.NOT_IMPLEMENTED_501,
+        null, "Sqoop version request not yet implemented");
+  }
+
   private Response getPigVersion() {
     return SimpleWebException.buildMessage(HttpStatus.NOT_IMPLEMENTED_501,
         null, "Pig version request not yet implemented");

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java?rev=1590179&r1=1590178&r2=1590179&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LogRetriever.java
Fri Apr 25 22:42:27 2014
@@ -127,6 +127,7 @@ public class LogRetriever {
     case HIVE:
       jobIDParser = new HiveJobIDParser(statusDir, conf);
       break;
+    case SQOOP:
     case JAR:
     case STREAMING:
       jobIDParser = new JarJobIDParser(statusDir, conf);
@@ -134,7 +135,7 @@ public class LogRetriever {
     default:
       System.err
         .println("Unknown job type:" + jobType!=null? jobType.toString():"null"
-        + ", only pig/hive/jar/streaming are supported, skip logs");
+        + ", only pig/hive/jar/streaming/sqoop are supported, skip logs");
       return;
     }
     List<String> jobs = new ArrayList<String>();



Mime
View raw message