Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C11EE200D44 for ; Mon, 20 Nov 2017 11:43:21 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id BF790160BF9; Mon, 20 Nov 2017 10:43:21 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B7345160BEC for ; Mon, 20 Nov 2017 11:43:20 +0100 (CET) Received: (qmail 51042 invoked by uid 500); 20 Nov 2017 10:43:19 -0000 Mailing-List: contact commits-help@ranger.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ranger.apache.org Delivered-To: mailing list commits@ranger.apache.org Received: (qmail 51033 invoked by uid 99); 20 Nov 2017 10:43:19 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 20 Nov 2017 10:43:19 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C3E43DFFB5; Mon, 20 Nov 2017 10:43:19 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@ranger.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ranger git commit: RANGER-1883: Part 2 - All tag sources and sinks should use the same kerberos identity and reuse it Date: Mon, 20 Nov 2017 10:43:19 +0000 (UTC) archived-at: Mon, 20 Nov 2017 10:43:21 -0000 Repository: ranger Updated Branches: refs/heads/master 6f4e28639 -> 10051777e RANGER-1883: Part 2 - All tag sources and sinks should use the same kerberos identity and reuse it Signed-off-by: Madhan Neethiraj Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/10051777 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/10051777 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/10051777 Branch: refs/heads/master Commit: 10051777edf21c5ab7914c0670e23040d105dc4c Parents: 6f4e286 Author: Abhay Kulkarni Authored: Mon Nov 13 18:56:03 2017 -0800 Committer: Madhan Neethiraj Committed: Mon Nov 20 02:43:09 2017 -0800 ---------------------------------------------------------------------- .../ranger/tagsync/process/TagSyncConfig.java | 6 ++ .../ranger/tagsync/process/TagSynchronizer.java | 94 ++++++++++++++++++-- .../tagsync/sink/tagadmin/TagAdminRESTSink.java | 52 +++-------- .../source/atlasrest/AtlasRESTTagSource.java | 30 ++----- .../tagsync/source/atlasrest/AtlasRESTUtil.java | 39 ++++---- 5 files changed, 132 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java index 3f35097..697c7cc 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java @@ -98,6 +98,8 @@ public class TagSyncConfig extends Configuration { private static final String TAGSYNC_KERBEROS_PRICIPAL = "ranger.tagsync.kerberos.principal"; private static final String TAGSYNC_KERBEROS_KEYTAB = "ranger.tagsync.kerberos.keytab"; + public static final String TAGSYNC_KERBEROS_IDENTITY = "tagsync.kerberos.identity"; + private static String LOCAL_HOSTNAME = "unknown"; private Properties props; @@ -399,6 +401,10 @@ public class TagSyncConfig extends Configuration { return ret; } + static public String getTagsyncKerberosIdentity(Properties prop) { + return prop.getProperty(TAGSYNC_KERBEROS_IDENTITY); + } + private TagSyncConfig() { super(false); init(); http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java index d36ecae..b07cd34 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java @@ -22,10 +22,14 @@ package org.apache.ranger.tagsync.process; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Logger; import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.model.TagSource; +import javax.security.auth.Subject; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -36,6 +40,8 @@ public class TagSynchronizer { private static final Logger LOG = Logger.getLogger(TagSynchronizer.class); + private static final String AUTH_TYPE_KERBEROS = "kerberos"; + private static final String TAGSYNC_SOURCE_BASE = "ranger.tagsync.source."; private static final String PROP_CLASS_NAME = "class"; @@ -97,15 +103,19 @@ public class TagSynchronizer { printConfigurationProperties(properties); - boolean ret = false; + boolean ret = initializeKerberosIdentity(properties); - LOG.info("Initializing TAG source and sink"); + if (ret) { + LOG.info("Initializing TAG source and sink"); - tagSink = initializeTagSink(properties); + tagSink = initializeTagSink(properties); - if (tagSink != null) { - initializeTagSources(); - ret = true; + if (tagSink != null) { + initializeTagSources(); + ret = true; + } + } else { + LOG.error("Error initializing kerberos identity"); } if (LOG.isDebugEnabled()) { @@ -344,6 +354,78 @@ public class TagSynchronizer { return tagSource; } + private static boolean initializeKerberosIdentity(Properties props) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagSynchronizer.initializeKerberosIdentity()"); + } + + boolean ret = false; + + String authenticationType = TagSyncConfig.getAuthenticationType(props); + String principal = TagSyncConfig.getKerberosPrincipal(props); + String keytab = TagSyncConfig.getKerberosKeytab(props); + String nameRules = TagSyncConfig.getNameRules(props); + + if (LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { + LOG.debug("authenticationType=" + authenticationType); + LOG.debug("principal=" + principal); + LOG.debug("keytab" + keytab); + LOG.debug("nameRules=" + nameRules); + } + } + final boolean isKerberized = !StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab); + + if (isKerberized) { + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to get kerberos identitiy"); + } + Subject subject = null; + try { + subject = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); + } catch(IOException exception) { + LOG.error("Could not get Subject from principal:[" + principal + "], keytab:[" + keytab + "], nameRules:[" + nameRules + "]", exception); + } + + UserGroupInformation kerberosIdentity; + + if (subject != null) { + try { + UserGroupInformation.loginUserFromSubject(subject); + kerberosIdentity = UserGroupInformation.getLoginUser(); + if (kerberosIdentity != null) { + props.put(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY, kerberosIdentity.getUserName()); + if (LOG.isDebugEnabled()) { + LOG.debug("Got UGI, user:[" + kerberosIdentity.getUserName() + "]"); + } + ret = true; + } else { + LOG.error("KerberosIdentity is null!"); + } + } catch (IOException exception) { + LOG.error("Failed to get UGI from Subject:[" + subject + "]", exception); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Not configured for Kerberos Authentication"); + } + props.remove(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY); + + ret = true; + } + + if (!ret) { + props.remove(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagSynchronizer.initializeKerberosIdentity() : " + ret); + } + + return ret; + } + private static String getStringProperty(Properties props, String propName) { String ret = null; http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java index 4f6761f..c34b6ea 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java @@ -26,7 +26,6 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.security.SecureClientLogin; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.tagsync.model.TagSink; @@ -34,7 +33,6 @@ import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServiceTags; import org.apache.ranger.tagsync.process.TagSyncConfig; -import javax.security.auth.Subject; import javax.servlet.http.HttpServletResponse; import java.io.IOException; @@ -55,18 +53,13 @@ public class TagAdminRESTSink implements TagSink, Runnable { private static final String REST_URL_IMPORT_SERVICETAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/importservicetags/"; - private static final String AUTH_TYPE_KERBEROS = "kerberos"; - private long rangerAdminConnectionCheckInterval; private RangerRESTClient tagRESTClient = null; + private boolean isKerberized; + private BlockingQueue uploadWorkItems; - - private String authenticationType; - private String principal; - private String keytab; - private String nameRules; private Thread myThread = null; @@ -83,47 +76,28 @@ public class TagAdminRESTSink implements TagSink, Runnable { String userName = TagSyncConfig.getTagAdminUserName(properties); String password = TagSyncConfig.getTagAdminPassword(properties); rangerAdminConnectionCheckInterval = TagSyncConfig.getTagAdminConnectionCheckInterval(properties); - authenticationType = TagSyncConfig.getAuthenticationType(properties); - nameRules = TagSyncConfig.getNameRules(properties); - principal = TagSyncConfig.getKerberosPrincipal(properties); - keytab = TagSyncConfig.getKerberosKeytab(properties); + isKerberized = TagSyncConfig.getTagsyncKerberosIdentity(properties) != null; + if (LOG.isDebugEnabled()) { LOG.debug("restUrl=" + restUrl); LOG.debug("sslConfigFile=" + sslConfigFile); LOG.debug("userName=" + userName); - LOG.debug("rangerAdminConnectionCheckInterval" + rangerAdminConnectionCheckInterval); + LOG.debug("rangerAdminConnectionCheckInterval=" + rangerAdminConnectionCheckInterval); + LOG.debug("isKerberized=" + isKerberized); } if (StringUtils.isNotBlank(restUrl)) { tagRESTClient = new RangerRESTClient(restUrl, sslConfigFile); - if(isKerberosEnabled()) { - Subject subject = null; - try { - subject = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); - } catch(IOException exception) { - LOG.error("Could not get Subject from principal:[" + principal + "], keytab:[" + keytab + "], nameRules:[" + nameRules + "]", exception); - } - if (subject != null) { - try { - UserGroupInformation.loginUserFromSubject(subject); - ret = true; - } catch (IOException exception) { - LOG.error("Failed to get UGI from Subject:[" + subject + "]"); - } - } - } else { + if(!isKerberized) { tagRESTClient.setBasicAuthInfo(userName, password); - ret = true; } + uploadWorkItems = new LinkedBlockingQueue(); + ret = true; } else { LOG.error("No value specified for property 'ranger.tagsync.tagadmin.rest.url'!"); } - if (ret) { - uploadWorkItems = new LinkedBlockingQueue(); - } - if(LOG.isDebugEnabled()) { LOG.debug("<== TagAdminRESTSink.initialize(), result=" + ret); } @@ -152,12 +126,8 @@ public class TagAdminRESTSink implements TagSink, Runnable { return ret; } - private boolean isKerberosEnabled() { - return !StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab); - } - private ServiceTags doUpload(ServiceTags serviceTags) throws Exception { - if(isKerberosEnabled()) { + if(isKerberized) { try{ UserGroupInformation userGroupInformation = UserGroupInformation.getLoginUser(); if (userGroupInformation != null) { @@ -170,7 +140,7 @@ public class TagAdminRESTSink implements TagSink, Runnable { } if (userGroupInformation != null) { if (LOG.isDebugEnabled()) { - LOG.debug("Using Principal = " + principal + ", keytab = " + keytab); + LOG.debug("Using Principal = " + userGroupInformation.getUserName()); } final ServiceTags serviceTag = serviceTags; ServiceTags ret = userGroupInformation.doAs(new PrivilegedAction() { http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index 7da4ed4..4e0ae90 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -29,7 +29,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.security.SecureClientLogin; import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.tagsync.model.AbstractTagSource; import org.apache.ranger.plugin.util.ServiceTags; @@ -47,16 +46,9 @@ import java.util.Properties; public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { private static final Log LOG = LogFactory.getLog(AtlasRESTTagSource.class); - static final String AUTH_TYPE_KERBEROS = "kerberos"; - private long sleepTimeBetweenCycleInMillis; - AtlasRESTUtil atlasRESTUtil = null; - - private String authenticationType; - private String principal; - private String keytab; - private String nameRules; + private AtlasRESTUtil atlasRESTUtil = null; private Thread myThread = null; @@ -103,30 +95,18 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { boolean ret = AtlasResourceMapperUtil.initializeAtlasResourceMappers(properties); sleepTimeBetweenCycleInMillis = TagSyncConfig.getTagSourceAtlasDownloadIntervalInMillis(properties); + final boolean isKerberized = TagSyncConfig.getTagsyncKerberosIdentity(properties) != null; String restUrl = TagSyncConfig.getAtlasRESTEndpoint(properties); String sslConfigFile = TagSyncConfig.getAtlasRESTSslConfigFile(properties); String userName = TagSyncConfig.getAtlasRESTUserName(properties); String password = TagSyncConfig.getAtlasRESTPassword(properties); - authenticationType = TagSyncConfig.getAuthenticationType(properties); - nameRules = TagSyncConfig.getNameRules(properties); - principal = TagSyncConfig.getKerberosPrincipal(properties); - keytab = TagSyncConfig.getKerberosKeytab(properties); - - final boolean kerberized = StringUtils.isNotEmpty(authenticationType) - && authenticationType.trim().equalsIgnoreCase(AtlasRESTTagSource.AUTH_TYPE_KERBEROS) - && SecureClientLogin.isKerberosCredentialExists(principal, keytab); - if (LOG.isDebugEnabled()) { LOG.debug("restUrl=" + restUrl); LOG.debug("sslConfigFile=" + sslConfigFile); LOG.debug("userName=" + userName); - LOG.debug("authenticationType=" + authenticationType); - LOG.debug("principal=" + principal); - LOG.debug("keytab=" + keytab); - LOG.debug("nameRules=" + nameRules); - LOG.debug("kerberized=" + kerberized); + LOG.debug("kerberized=" + isKerberized); } if (StringUtils.isNotEmpty(restUrl)) { @@ -135,10 +115,10 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { } RangerRESTClient atlasRESTClient = new RangerRESTClient(restUrl, sslConfigFile); - if (!kerberized) { + if (!isKerberized) { atlasRESTClient.setBasicAuthInfo(userName, password); } - atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, kerberized, authenticationType, principal, keytab, nameRules); + atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, isKerberized); } else { LOG.info("AtlasEndpoint not specified, Initial download of Atlas-entities cannot be done."); ret = false; http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java index 167fe68..00a101e 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,14 +28,14 @@ import org.apache.atlas.typesystem.json.InstanceSerialization; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.security.SecureClientLogin; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.log4j.Logger; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.plugin.util.RangerRESTClient; import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits; import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil; -import javax.security.auth.Subject; +import java.io.IOException; import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.HashMap; @@ -66,22 +66,17 @@ public class AtlasRESTUtil { private final Gson gson = new Gson(); private final RangerRESTClient atlasRESTClient; - private final String principal; - private final String keytab; - private final String nameRules; - private final boolean kerberized; - public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean kerberized, String authenticationType, String principal, String keytab, String nameRules) { + private final boolean isKerberized; + + public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean isKerberized) { if (LOG.isDebugEnabled()) { LOG.debug("==> AtlasRESTUtil()"); } - this.kerberized = kerberized; - this.atlasRESTClient = atlasRESTClient; - this.principal = principal; - this.keytab = keytab; - this.nameRules = nameRules; + + this.isKerberized = isKerberized; if (LOG.isDebugEnabled()) { LOG.debug("<== AtlasRESTUtil()"); @@ -249,13 +244,23 @@ public class AtlasRESTUtil { Map ret = new HashMap(); try { - if (kerberized) { + UserGroupInformation userGroupInformation = null; + if (isKerberized) { + userGroupInformation = UserGroupInformation.getLoginUser(); + + try { + userGroupInformation.checkTGTAndReloginFromKeytab(); + } catch (IOException ioe) { + LOG.error("Error renewing TGT and relogin", ioe); + userGroupInformation = null; + } + } + if (userGroupInformation != null) { LOG.debug("Using kerberos authentication"); - Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules); if(LOG.isDebugEnabled()) { - LOG.debug("Using Principal = "+ principal + ", keytab = "+keytab); + LOG.debug("Using Principal = "+ userGroupInformation.getUserName()); } - ret = Subject.doAs(sub, new PrivilegedAction>() { + ret = userGroupInformation.doAs(new PrivilegedAction>() { @Override public Map run() { try{