incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1166830 - in /incubator/hcatalog/trunk: CHANGES.txt src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
Date Thu, 08 Sep 2011 18:15:32 GMT
Author: hashutosh
Date: Thu Sep  8 18:15:32 2011
New Revision: 1166830

URL: http://svn.apache.org/viewvc?rev=1166830&view=rev
Log:
HCATALOG-94: Handle the job tracker tokens properly in har jobs

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1166830&r1=1166829&r2=1166830&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Sep  8 18:15:32 2011
@@ -25,6 +25,8 @@ Trunk (unreleased changes)
   NEW FEATURES
 
   IMPROVEMENTS
+  HCAT-94. Handle the job tracker tokens properly in har jobs (khorgath via hashutosh)
+
   HCAT-43. hcat.sh does not pick up hcatalog.jar correctly (macyang via hashutosh)
   
   HCAT-91. hcat client not setting the config properly (hashutosh)

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java?rev=1166830&r1=1166829&r2=1166830&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java
Thu Sep  8 18:15:32 2011
@@ -171,13 +171,13 @@ public class HCatOutputCommitter extends
             (HCatConstants.HCAT_KEY_TOKEN_SIGNATURE) != null) {
           client.cancelDelegationToken(tokenStrForm);
         }
-        
-        String jcTokenStrForm = jobContext.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
-        String jcTokenSignature = jobContext.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
-        if(jcTokenStrForm != null && jcTokenSignature != null) {
-          HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
+        if (harProcessor.isEnabled()){
+          String jcTokenStrForm = jobContext.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
+          String jcTokenSignature = jobContext.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
+          if(jcTokenStrForm != null && jcTokenSignature != null) {
+            HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
+          }
         }
-        
       } catch(Exception e) {
         if( e instanceof HCatException ) {
           throw (HCatException) e;
@@ -376,12 +376,14 @@ public class HCatOutputCommitter extends
           client.cancelDelegationToken(tokenStrForm);
         }
 
-        String jcTokenStrForm = 
-            context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
-        String jcTokenSignature = 
-            context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
-        if(jcTokenStrForm != null && jcTokenSignature != null) {
-          HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
+        if (harProcessor.isEnabled()){
+          String jcTokenStrForm = 
+              context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
+          String jcTokenSignature = 
+              context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
+          if(jcTokenStrForm != null && jcTokenSignature != null) {
+            HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
+          }
         }
 
       } catch (Exception e) {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java?rev=1166830&r1=1166829&r2=1166830&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
(original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
Thu Sep  8 18:15:32 2011
@@ -248,9 +248,11 @@ public class HCatOutputFormat extends HC
             }
 
             String jcTokenSignature = "jc."+tokenSignature;
-            if(tokenMap.get(jcTokenSignature) == null) {
-              tokenMap.put(jcTokenSignature,
-                  HCatUtil.getJobTrackerDelegationToken(conf,ugi.getUserName()));
+            if (harRequested){
+              if(tokenMap.get(jcTokenSignature) == null) {
+                tokenMap.put(jcTokenSignature,
+                    HCatUtil.getJobTrackerDelegationToken(conf,ugi.getUserName()));
+              }
             }
             
             job.getCredentials().addToken(new Text(ugi.getUserName() + tokenSignature),
@@ -258,15 +260,20 @@ public class HCatOutputFormat extends HC
             // this will be used by the outputcommitter to pass on to the metastore client
             // which in turn will pass on to the TokenSelector so that it can select
             // the right token.
-            job.getCredentials().addToken(new Text(ugi.getUserName() + jcTokenSignature),
-                tokenMap.get(jcTokenSignature));
-            
             job.getConfiguration().set(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE, tokenSignature);
-            job.getConfiguration().set(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE,
jcTokenSignature);
-            job.getConfiguration().set(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM, tokenMap.get(jcTokenSignature).encodeToUrlString());
 
-//            LOG.info("Set hive dt["+tokenSignature+"]");
-//            LOG.info("Set jt dt["+jcTokenSignature+"]");
+            if (harRequested){
+              job.getCredentials().addToken(new Text(ugi.getUserName() + jcTokenSignature),
+                  tokenMap.get(jcTokenSignature));
+
+              job.getConfiguration().set(
+                  HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE, jcTokenSignature);
+              job.getConfiguration().set(
+                  HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM, 
+                  tokenMap.get(jcTokenSignature).encodeToUrlString());
+              //          LOG.info("Set hive dt["+tokenSignature+"]");
+              //          LOG.info("Set jt dt["+jcTokenSignature+"]");
+            }
           }
        }
       } catch(Exception e) {



Mime
View raw message