Return-Path: Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: (qmail 78561 invoked from network); 5 Apr 2011 18:50:38 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 5 Apr 2011 18:50:38 -0000 Received: (qmail 16836 invoked by uid 500); 5 Apr 2011 18:50:38 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 16794 invoked by uid 500); 5 Apr 2011 18:50:38 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 16785 invoked by uid 99); 5 Apr 2011 18:50:38 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Apr 2011 18:50:38 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Apr 2011 18:50:36 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 3D0672388900; Tue, 5 Apr 2011 18:50:16 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1089157 - in /hadoop/common/branches/branch-0.20-security: ./ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/hdfs/org/apache/hadoop/hdfs/tools/ src/mapred/org/apache/hadoop/mapreduce/security/token/ Date: Tue, 05 Apr 2011 18:50:16 -0000 To: common-commits@hadoop.apache.org From: boryas@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110405185016.3D0672388900@eris.apache.org> Author: boryas Date: Tue Apr 5 18:50:15 2011 New Revision: 1089157 URL: http://svn.apache.org/viewvc?rev=1089157&view=rev Log: MAPREDUCE-2420. JobTracker should be able to renew delegation token over HTTP Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1089157&r1=1089156&r2=1089157&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Tue Apr 5 18:50:15 2011 @@ -40,7 +40,8 @@ Release 0.20.204.0 - unreleased HDFS-1692. In secure mode, Datanode process doesn't exit when disks fail. (bharathm via boryas) - + MAPREDUCE-2420. JobTracker should be able to renew delegation token + over HTTP (boryas) IMPROVEMENTS HDFS-1541. Not marking datanodes dead when namenode in safemode. Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java?rev=1089157&r1=1089156&r2=1089157&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java (original) +++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java Tue Apr 5 18:50:15 2011 @@ -77,9 +77,14 @@ public class RenewDelegationTokenServlet os.println(result); os.close(); } catch(Exception e) { - LOG.info("Exception while renewing token. Re-throwing. ", e); + // transfer exception over the http + String exceptionClass = e.getClass().getCanonicalName(); + String exceptionMsg = e.getLocalizedMessage(); + String strException = exceptionClass + ";" + exceptionMsg; + LOG.info("Exception while renewing token. Re-throwing. s=" + strException, e); + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - e.getMessage()); + strException); } } } Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1089157&r1=1089156&r2=1089157&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original) +++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Tue Apr 5 18:50:15 2011 @@ -198,21 +198,56 @@ public class DelegationTokenFetcher { buf.append("="); buf.append(tok.encodeToUrlString()); BufferedReader in = null; + HttpURLConnection connection = null; try { URL url = new URL(buf.toString()); SecurityUtil.fetchServiceTicket(url); - URLConnection connection = url.openConnection(); + connection = (HttpURLConnection)url.openConnection(); in = new BufferedReader(new InputStreamReader (connection.getInputStream())); long result = Long.parseLong(in.readLine()); in.close(); return result; } catch (IOException ie) { + LOG.info("error in renew over HTTP", ie); + IOException e = null; + if(connection != null) { + String resp = connection.getResponseMessage(); + e = getExceptionFromResponse(resp); + } + IOUtils.cleanup(LOG, in); + if(e!=null) { + LOG.info("rethrowing exception from HTTP request: " + e.getLocalizedMessage()); + throw e; + } throw ie; } } + static private IOException getExceptionFromResponse(String resp) { + String exceptionClass = "", exceptionMsg = ""; + if(resp != null && !resp.isEmpty()) { + String[] rs = resp.split(";"); + exceptionClass = rs[0]; + exceptionMsg = rs[1]; + } + LOG.info("Error response from HTTP request=" + resp + + ";ec=" + exceptionClass + ";em="+exceptionMsg); + IOException e = null; + if(exceptionClass != null && !exceptionClass.isEmpty()) { + if(exceptionClass.contains("InvalidToken")) { + e = new org.apache.hadoop.security.token.SecretManager.InvalidToken(exceptionMsg); + e.setStackTrace(new StackTraceElement[0]); // stack is not relevant + } else if(exceptionClass.contains("AccessControlException")) { + e = new org.apache.hadoop.security.AccessControlException(exceptionMsg); + e.setStackTrace(new StackTraceElement[0]); // stack is not relevant + } + } + LOG.info("Exception from HTTP response=" + e.getLocalizedMessage()); + return e; + } + /** * Cancel a Delegation Token. * @param nnAddr the NameNode's address Modified: hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java?rev=1089157&r1=1089156&r2=1089157&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java (original) +++ hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java Tue Apr 5 18:50:15 2011 @@ -25,13 +25,11 @@ import org.apache.hadoop.security.Access import org.apache.hadoop.security.UserGroupInformation; import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.Iterator; -import java.util.List; import java.util.Set; import java.util.Timer; import java.util.TimerTask; @@ -42,7 +40,9 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.VersionMismatchException; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; @@ -143,6 +143,21 @@ public class DelegationTokenRenewal { } } + protected static long renewDelegationTokenOverHttps( + final Token token, final Configuration conf) + throws InterruptedException, IOException{ + final String httpAddress = getHttpAddressForToken(token, conf); + + Long expDate = (Long) UserGroupInformation.getLoginUser().doAs( + new PrivilegedExceptionAction() { + public Long run() throws IOException { + return DelegationTokenFetcher.renewDelegationToken(httpAddress, token); + } + }); + LOG.info("Renew over HTTP done. addr="+httpAddress+";res="+expDate); + return expDate; + } + private static long renewDelegationToken(DelegationTokenToRenew dttr) throws Exception { long newExpirationDate=System.currentTimeMillis()+3600*1000; @@ -151,10 +166,17 @@ public class DelegationTokenRenewal { if(token.getKind().equals(kindHdfs)) { try { - DistributedFileSystem dfs = getDFSForToken(token, conf); - newExpirationDate = dfs.renewDelegationToken(token); - } catch (InvalidToken ite) { - LOG.warn("invalid token - not scheduling for renew"); + try { + DistributedFileSystem dfs = getDFSForToken(token, conf); + newExpirationDate = dfs.renewDelegationToken(token); + } catch(VersionMismatchException vme) { + // if there is a version mismatch we try over https + LOG.info("Delegation token renew for t=" + token.getService() + + " failed with VersionMissmaptch:" + vme.toString()+". Trying over https"); + renewDelegationTokenOverHttps(token, conf); + } + }catch (InvalidToken ite) { + LOG.warn("invalid token - not scheduling for renew: " + ite.getLocalizedMessage()); removeFailedDelegationToken(dttr); throw new IOException("failed to renew token", ite); } catch (AccessControlException ioe) { @@ -166,7 +188,7 @@ public class DelegationTokenRenewal { // returns default expiration date } } else { - throw new Exception("unknown token type to renew+"+token.getKind()); + throw new Exception("unknown token type to renew: "+token.getKind()); } return newExpirationDate; } @@ -200,6 +222,20 @@ public class DelegationTokenRenewal { } } + private static String getHttpAddressForToken( + Token token, final Configuration conf) + throws IOException { + + String[] ipaddr = token.getService().toString().split(":"); + + InetAddress iaddr = InetAddress.getByName(ipaddr[0]); + String dnsName = iaddr.getCanonicalHostName(); + String httpsPort = conf.get("dfs.https.port", "50470"); + + // always use https (it is for security only) + return "https://" + dnsName+":"+httpsPort; + } + private static DistributedFileSystem getDFSForToken( Token token, final Configuration conf) throws Exception {