hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1584196 - in /hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton: HiveDelegator.java tool/JobSubmissionConstants.java tool/LaunchMapper.java
Date Wed, 02 Apr 2014 21:18:08 GMT
Author: thejas
Date: Wed Apr  2 21:18:08 2014
New Revision: 1584196

URL: http://svn.apache.org/r1584196
Log:
HIVE-6780 : Set tez credential file property along with MR conf property for Tez jobs (Eugene
Koifman via Thejas Nair)

Modified:
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
Wed Apr  2 21:18:08 2014
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.exec.ExecuteException;
+import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants;
 import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
 import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
 
@@ -76,6 +77,11 @@ public class HiveDelegator extends Launc
       args.add("--hiveconf");
       args.add(TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);
 
+      //this is needed specifcally for Hive on Tez (in addition to
+      //JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER)
+      args.add("--hiveconf");
+      args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ);
+
       for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) {
         args.add("--hiveconf");
         args.add(TempletonUtils.quoteForWindows(prop));

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
Wed Apr  2 21:18:08 2014
@@ -41,6 +41,8 @@ public interface JobSubmissionConstants 
    */
   public static final String TOKEN_FILE_ARG_PLACEHOLDER =
     "__MR_JOB_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION__";
+  public static final String TOKEN_FILE_ARG_PLACEHOLDER_TEZ =
+    "__TEZ_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION_TEZ__";
   /**
    * constants needed for Pig job submission
    */

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1584196&r1=1584195&r2=1584196&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
Wed Apr  2 21:18:08 2014
@@ -100,41 +100,47 @@ public class LaunchMapper extends Mapper
     Map<String, String> env = TempletonUtils.hadoopUserEnv(user, overrideClasspath);
     handlePigEnvVars(conf, env);
     List<String> jarArgsList = new LinkedList<String>(Arrays.asList(jarArgs));
-    String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
-
+    handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary");
+    handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path");
+    boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ?
+            false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS));
+    return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps);
+  }
 
+  /**
+   * Replace placeholder with actual "prop=file".  This is done multiple times (possibly)
since
+   * Tez and MR use different property names
+   */
+  private static void handleTokenFile(List<String> jarArgsList, String tokenPlaceHolder,
String tokenProperty) throws IOException {
+    String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
     if (tokenFile != null) {
       //Token is available, so replace the placeholder
       tokenFile = tokenFile.replaceAll("\"", "");
-      String tokenArg = "mapreduce.job.credentials.binary=" + tokenFile;
+      String tokenArg = tokenProperty + "=" + tokenFile;
       if (Shell.WINDOWS) {
         try {
           tokenArg = TempletonUtils.quoteForWindows(tokenArg);
         } catch (BadParam e) {
-          String msg = "cannot pass " + tokenFile + " to mapreduce.job.credentials.binary";
+          String msg = "cannot pass " + tokenFile + " to " + tokenProperty;
           LOG.error(msg, e);
           throw new IOException(msg, e);
         }
       }
       for(int i=0; i<jarArgsList.size(); i++){
         String newArg =
-                jarArgsList.get(i).replace(TOKEN_FILE_ARG_PLACEHOLDER, tokenArg);
+          jarArgsList.get(i).replace(tokenPlaceHolder, tokenArg);
         jarArgsList.set(i, newArg);
       }
-
     }else{
       //No token, so remove the placeholder arg
       Iterator<String> it = jarArgsList.iterator();
       while(it.hasNext()){
         String arg = it.next();
-        if(arg.contains(TOKEN_FILE_ARG_PLACEHOLDER)){
+        if(arg.contains(tokenPlaceHolder)){
           it.remove();
         }
       }
     }
-    boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ?
-            false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS));
-    return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps);
   }
 
   private void copyLocal(String var, Configuration conf) throws IOException {



Mime
View raw message