ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From v..@apache.org
Subject incubator-ranger git commit: Issue RANGER-827
Date Thu, 11 Feb 2016 22:49:38 GMT
Repository: incubator-ranger
Updated Branches:
  refs/heads/master 0966bbccb -> 658f2310d


Issue RANGER-827

Signed-off-by: Velmurugan Periasamy <vel@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/658f2310
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/658f2310
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/658f2310

Branch: refs/heads/master
Commit: 658f2310dc674391b7d930e4d3a43cc976693370
Parents: 0966bbc
Author: Bolke de Bruin <bolke@xs4all.nl>
Authored: Sun Feb 7 22:48:07 2016 +0100
Committer: Velmurugan Periasamy <vel@apache.org>
Committed: Thu Feb 11 17:49:06 2016 -0500

----------------------------------------------------------------------
 .../config/UserGroupSyncConfig.java             |  67 ++-
 .../process/UnixUserGroupBuilder.java           | 454 +++++++++++++------
 .../process/UnixUserGroupBuilderTest.java       | 108 +++++
 3 files changed, 481 insertions(+), 148 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/658f2310/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
index e46b469..9bcf767 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/config/UserGroupSyncConfig.java
@@ -58,6 +58,8 @@ public class UserGroupSyncConfig  {
 	public static final String  UGSYNC_PM_URL_PROP = 	"ranger.usersync.policymanager.baseURL"
;
 	
 	public static final String  UGSYNC_MIN_USERID_PROP  = 	"ranger.usersync.unix.minUserId"
;
+
+	public static final String  UGSYNC_MIN_GROUPID_PROP =   "ranger.usersync.unix.minGroupId"
;
 	
 	public static final String  UGSYNC_MAX_RECORDS_PER_API_CALL_PROP  = 	"ranger.usersync.policymanager.maxrecordsperapicall"
;
 
@@ -162,6 +164,16 @@ public class UserGroupSyncConfig  {
   private static final String LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "ranger.usersync.group.memberattributename";
   private static final String DEFAULT_LGSYNC_GROUP_MEMBER_ATTRIBUTE_NAME = "member";
 
+	private static final String UGSYNC_UPDATE_MILLIS_MIN = "ranger.usersync.unix.updatemillismin";
+	private final static long DEFAULT_UGSYNC_UPDATE_MILLIS_MIN = 1 * 60 * 1000; // ms
+
+	private static final String UGSYNC_UNIX_BACKEND = "ranger.usersync.unix.backend";
+	private final static String DEFAULT_UGSYNC_UNIX_BACKEND = "passwd";
+
+	private static final String UGSYNC_GROUP_ENUMERATE_ENABLED = "ranger.usersync.group.enumerate";
+
+	private static final String UGSYNC_GROUP_ENUMERATE_GROUPS = "ranger.usersync.group.enumerategroup";
+
 	private static final String SYNC_POLICY_MGR_KEYSTORE = "ranger.usersync.policymgr.keystore";
 
 	private static final String SYNC_POLICY_MGR_ALIAS = "ranger.usersync.policymgr.alias";
@@ -192,7 +204,7 @@ public class UserGroupSyncConfig  {
 	private static volatile UserGroupSyncConfig me = null ;
 	
 	public static UserGroupSyncConfig getInstance() {
-        UserGroupSyncConfig result = me;
+		UserGroupSyncConfig result = me;
 		if (result == null) {
 			synchronized(UserGroupSyncConfig.class) {
 				result = me ;
@@ -204,11 +216,10 @@ public class UserGroupSyncConfig  {
 		return result ;
 	}
 	
-	
 	private UserGroupSyncConfig() {
-		init() ;
+		init();
 	}
-	
+
 	private void init() {
 		readConfigFile(CONFIG_FILE);
 		readConfigFile(DEFAULT_CONFIG_FILE);
@@ -321,19 +332,35 @@ public class UserGroupSyncConfig  {
 		}
 		return val;
 	}
-	
+
+	public String getUnixBackend() {
+		String val = prop.getProperty(UGSYNC_UNIX_BACKEND);
+		if ( val == null ) {
+			val = DEFAULT_UGSYNC_UNIX_BACKEND;
+		}
+
+		return val;
+	}
+
 	public boolean isUserSyncEnabled() {
 		String val = prop.getProperty(UGSYNC_ENABLED_PROP) ;
 		return (val != null && val.trim().equalsIgnoreCase("true")) ;
 	}
 
-	
+	public String getEnumerateGroups() {
+		return prop.getProperty(UGSYNC_GROUP_ENUMERATE_GROUPS);
+	}
+
+	public boolean isGroupEnumerateEnabled() {
+		String val = prop.getProperty(UGSYNC_GROUP_ENUMERATE_ENABLED) ;
+		return (val != null && val.trim().equalsIgnoreCase("true")) ;
+	}
+
 	public boolean isMockRunEnabled() {
 		String val = prop.getProperty(UGSYNC_MOCK_RUN_PROP) ;
 		return (val != null && val.trim().equalsIgnoreCase("true")) ;
 	}
-	
-	
+
 	public String getPolicyManagerBaseURL() {
 		return prop.getProperty(UGSYNC_PM_URL_PROP) ;
 	}
@@ -342,6 +369,8 @@ public class UserGroupSyncConfig  {
 	public String getMinUserId() {
 		return prop.getProperty(UGSYNC_MIN_USERID_PROP) ;
 	}
+
+	public String getMinGroupId() { return prop.getProperty(UGSYNC_MIN_GROUPID_PROP) ; }
 	
 	public String getMaxRecordsPerAPICall() {
 		return prop.getProperty(UGSYNC_MAX_RECORDS_PER_API_CALL_PROP) ;
@@ -366,7 +395,20 @@ public class UserGroupSyncConfig  {
 		return  prop.getProperty(SSL_TRUSTSTORE_PATH_PASSWORD_PARAM) ;
 	}
 	
-	
+	public long getUpdateMillisMin() {
+		String val = prop.getProperty(UGSYNC_UPDATE_MILLIS_MIN) ;
+		if (val == null) {
+			return DEFAULT_UGSYNC_UPDATE_MILLIS_MIN ;
+		}
+
+		long ret = Long.parseLong(val) ;
+		if (ret < DEFAULT_UGSYNC_UPDATE_MILLIS_MIN) {
+			return DEFAULT_UGSYNC_UPDATE_MILLIS_MIN;
+		}
+
+		return ret;
+	}
+
 	public long getSleepTimeInMillisBetweenCycle() throws Throwable {
 		String val =  prop.getProperty(UGSYNC_SLEEP_TIME_IN_MILLIS_BETWEEN_CYCLE_PARAM) ;
 		if (val == null) {
@@ -789,7 +831,7 @@ public class UserGroupSyncConfig  {
 		}
 		return val;
 	}
-	
+
 	/* Used only for unit testing */
     public void setUserSearchFilter(String filter) {
             prop.setProperty(LGSYNC_USER_SEARCH_FILTER, filter);
@@ -809,4 +851,9 @@ public class UserGroupSyncConfig  {
     public void setPagedResultsEnabled(boolean pagedResultsEnabled) {
         prop.setProperty(LGSYNC_PAGED_RESULTS_ENABLED, String.valueOf(pagedResultsEnabled));
     }
+
+	/* Used only for unit testing */
+	public void setProperty(String name, String value) {
+		prop.setProperty(name, value);
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/658f2310/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
----------------------------------------------------------------------
diff --git a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
index f0a29f4..780f670 100644
--- a/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
+++ b/ugsync/src/main/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilder.java
@@ -17,16 +17,15 @@
  * under the License.
  */
 
- package org.apache.ranger.unixusersync.process;
+package org.apache.ranger.unixusersync.process;
 
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.io.InputStreamReader;
+import java.util.*;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.log4j.Logger;
 import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
 import org.apache.ranger.usergroupsync.UserGroupSink;
@@ -35,51 +34,94 @@ import org.apache.ranger.usergroupsync.UserGroupSource;
 public class UnixUserGroupBuilder implements UserGroupSource {
 	
 	private static final Logger LOG = Logger.getLogger(UnixUserGroupBuilder.class) ;
+	private final static String OS = System.getProperty("os.name") ;
 
+	// kept for legacy support
 	public static final String UNIX_USER_PASSWORD_FILE = "/etc/passwd" ;
 	public static final String UNIX_GROUP_FILE = "/etc/group" ;
 
+	/** Shell commands to get users and groups */
+	static final String LINUX_GET_ALL_USERS_CMD = "getent passwd" ;
+	static final String LINUX_GET_ALL_GROUPS_CMD = "getent group" ;
+	static final String LINUX_GET_GROUP_CMD = "getent group %s" ;
+
+	// mainly for testing purposes
+	// there might be a better way
+	static final String MAC_GET_ALL_USERS_CMD = "dscl . -readall /Users UniqueID PrimaryGroupID
| " +
+			"awk 'BEGIN { OFS = \":\"; ORS=\"\\n\"; i=0;}" +
+			"/RecordName: / {name = $2;i = 0;}/PrimaryGroupID: / {gid = $2;}" +
+			"/^ / {if (i == 0) { i++; name = $1;}}" +
+			"/UniqueID: / {uid = $2;print name, \"*\", gid, uid;}'" ;
+	static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID | " +
+			"awk -v OFS=\":\" '{print $1, \"*\", $2, \"\"}'" ;
+	static final String MAC_GET_GROUP_CMD = "dscl . -read /Groups/%1$s | paste -d, -s - | sed
-e 's/:/|/g' | " +
+			"awk -v OFS=\":\" -v ORS=\"\\n\" -F, '{print \"%1$s\",\"*\",$6,$4}' | " +
+			"sed -e 's/:[^:]*| /:/g' | sed -e 's/ /,/g'" ;
+
+	static final String BACKEND_PASSWD = "passwd";
+
+	private boolean enumerateGroupMembers = false;
+	private boolean useNss = false;
+
+	private long lastUpdateTime = 0; // Last time maps were updated
+	private long timeout = 0;
+
 	private UserGroupSyncConfig config = UserGroupSyncConfig.getInstance() ;
 	private Map<String,List<String>>  	user2GroupListMap = new HashMap<String,List<String>>();
 	private Map<String,List<String>>  	internalUser2GroupListMap = new HashMap<String,List<String>>();
 	private Map<String,String>			groupId2groupNameMap = new HashMap<String,String>()
;
 	private int 						minimumUserId  = 0 ;
-	
-	private long  passwordFileModiiedAt = 0 ;
-	private long  groupFileModifiedAt = 0 ;
-		
-	
+	private int							minimumGroupId = 0 ;
+
+	private long passwordFileModifiedAt = 0 ;
+	private long groupFileModifiedAt = 0 ;
+
 	public static void main(String[] args) throws Throwable {
-		UnixUserGroupBuilder  ugbuilder = new UnixUserGroupBuilder() ;
+		UnixUserGroupBuilder ugbuilder = new UnixUserGroupBuilder() ;
 		ugbuilder.init();
 		ugbuilder.print(); 
 	}
 	
 	public UnixUserGroupBuilder() {
 		minimumUserId = Integer.parseInt(config.getMinUserId()) ;
-		LOG.debug("Minimum UserId: " + minimumUserId) ;
+		minimumGroupId = Integer.parseInt(config.getMinGroupId()) ;
+
+		LOG.debug("Minimum UserId: " + minimumUserId + ", minimum GroupId: " + minimumGroupId)
;
+
+		timeout = config.getUpdateMillisMin() ;
+		enumerateGroupMembers = config.isGroupEnumerateEnabled();
+
+		if (!config.getUnixBackend().equalsIgnoreCase(BACKEND_PASSWD)) {
+			useNss = true;
+		} else {
+			LOG.warn("DEPRECATED: Unix backend is configured to use /etc/passwd and /etc/group files
directly " +
+					"instead of standard system mechanisms.");
+		}
 	}
 
 	@Override
 	public void init() throws Throwable {
 		buildUserGroupInfo() ;
 	}
-	
+
 	@Override
 	public boolean isChanged() {
-		long TempPasswordFileModiiedAt = new File(UNIX_USER_PASSWORD_FILE).lastModified() ;
-		if (passwordFileModiiedAt != TempPasswordFileModiiedAt) {
+		if (useNss)
+			return System.currentTimeMillis() - lastUpdateTime > timeout ;
+
+		long TempPasswordFileModifiedAt = new File(UNIX_USER_PASSWORD_FILE).lastModified() ;
+		if (passwordFileModifiedAt != TempPasswordFileModifiedAt) {
 			return true ;
 		}
-		
+
 		long TempGroupFileModifiedAt = new File(UNIX_GROUP_FILE).lastModified() ;
 		if (groupFileModifiedAt != TempGroupFileModifiedAt) {
 			return true ;
 		}
-		
+
 		return false ;
 	}
-	
+
 
 	@Override
 	public void updateSink(UserGroupSink sink) throws Throwable {
@@ -96,10 +138,21 @@ public class UnixUserGroupBuilder implements UserGroupSource {
 	
 	private void buildUserGroupInfo() throws Throwable {
 		user2GroupListMap = new HashMap<String,List<String>>();
-		groupId2groupNameMap = new HashMap<String,String>() ;
+		groupId2groupNameMap = new HashMap<String, String>();
+
+		if (OS.startsWith("Mac")) {
+			buildUnixGroupList(MAC_GET_ALL_GROUPS_CMD, MAC_GET_GROUP_CMD, false);
+			buildUnixUserList(MAC_GET_ALL_USERS_CMD);
+		} else {
+			if (!OS.startsWith("Linux")) {
+				LOG.warn("Platform not recognized assuming Linux compatible");
+			}
+			buildUnixGroupList(LINUX_GET_ALL_GROUPS_CMD, LINUX_GET_GROUP_CMD, true);
+			buildUnixUserList(LINUX_GET_ALL_USERS_CMD);
+		}
+
+		lastUpdateTime = System.currentTimeMillis() ;
 
-		buildUnixGroupList(); 
-		buildUnixUserList();
 		if (LOG.isDebugEnabled()) {
 			print() ;
 		}
@@ -117,139 +170,264 @@ public class UnixUserGroupBuilder implements UserGroupSource {
 		}
 	}
 	
-	private void buildUnixUserList() throws Throwable {
-		
-		File f = new File(UNIX_USER_PASSWORD_FILE) ;
-
-		if (f.exists()) {
-			
-			
-			BufferedReader reader = null ;
-			
-			reader = new BufferedReader(new FileReader(f)) ;
-			
-			String line = null ;
-			
-			while ( (line = reader.readLine()) != null) {
-				
-				if (line.trim().isEmpty()) 
-					continue ;
-				
-				String[] tokens = line.split(":") ;
-				
-				int len = tokens.length ;
-				
-				if (len < 2) {
-					continue ;
+	private void buildUnixUserList(String command) throws Throwable {
+		BufferedReader reader = null;
+
+		if (!useNss) {
+			File file = new File(UNIX_USER_PASSWORD_FILE);
+			passwordFileModifiedAt = file.lastModified();
+			reader = new BufferedReader(new FileReader(file)) ;
+		} else {
+			Process process = Runtime.getRuntime().exec(
+					new String[]{"bash", "-c", command});
+
+			reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+		}
+
+		String line = null;
+		Map<String,String> userName2uid = new HashMap<String,String>();
+
+		while ((line = reader.readLine()) != null) {
+			if (line.trim().isEmpty())
+				continue;
+
+			String[] tokens = line.split(":");
+
+			int len = tokens.length;
+
+			if (len < 2) {
+				continue;
+			}
+
+			String userName = tokens[0];
+			String userId = tokens[2];
+			String groupId = tokens[3];
+
+			int numUserId = -1;
+
+			try {
+				numUserId = Integer.parseInt(userId);
+			} catch (NumberFormatException nfe) {
+				LOG.warn("Unix UserId: [" + userId + "]: can not be parsed as valid int. considering
as  -1.", nfe);
+				numUserId = -1;
+			}
+
+			if (numUserId >= minimumUserId) {
+				userName2uid.put(userName, userId);
+				String groupName = groupId2groupNameMap.get(groupId);
+				if (groupName != null) {
+					List<String> groupList = new ArrayList<String>();
+					groupList.add(groupName);
+					// do we already know about this use's membership to other groups?  If so add those,
too
+					if (internalUser2GroupListMap.containsKey(userName)) {
+						List<String> map = internalUser2GroupListMap.get(userName);
+
+						// there could be duplicates
+						map.remove(groupName);
+						groupList.addAll(map);
+					}
+					user2GroupListMap.put(userName, groupList);
+				} else {
+					// we are ignoring the possibility that this user was present in /etc/groups.
+					LOG.warn("Group Name could not be found for group id: [" + groupId + "]. Skipping adding
user [" + userName + "] with id [" + userId + "].");
 				}
-				
-				String userName = tokens[0] ;
-				String userId = tokens[2] ;
-				String groupId = tokens[3] ;
-				
-				int numUserId = -1 ; 
-				
+			} else {
+				LOG.debug("Skipping user [" + userName + "] since its userid [" + userId + "] is less
than minuserid limit [" + minimumUserId + "].");
+			}
+		}
+
+		reader.close();
+
+		if (!useNss)
+			return;
+
+		// this does a reverse check as not all users might be listed in getent passwd
+		if (enumerateGroupMembers) {
+			LOG.debug("Start drill down group members");
+			for (Map.Entry<String, List<String>> entry : internalUser2GroupListMap.entrySet())
{
+				// skip users we already now about
+				if (user2GroupListMap.containsKey(entry.getKey()))
+					continue;
+
+				LOG.debug("Enumerating user " + entry.getKey());
+
+				int numUserId = -1;
 				try {
-					numUserId = Integer.parseInt(userId) ;
-				}
-				catch(NumberFormatException nfe) {
-					LOG.warn("Unix UserId: [" + userId + "]: can not be parsed as valid int. considering
as  -1.", nfe);
-					numUserId = -1 ;
+					numUserId = Integer.parseInt(userName2uid.get(entry.getKey()));
+				} catch (NumberFormatException nfe) {
+					numUserId = -1;
 				}
-									
-				if (numUserId >= minimumUserId ) {
-					String groupName = groupId2groupNameMap.get(groupId) ;
-					if (groupName != null) {
-						List<String> groupList = new ArrayList<String>();
-						groupList.add(groupName);
-						// do we already know about this use's membership to other groups?  If so add those,
too
-						if (internalUser2GroupListMap.containsKey(userName)) {
-							groupList.addAll(internalUser2GroupListMap.get(userName));
-						}
-						user2GroupListMap.put(userName, groupList);
-					}
-					else {
-						// we are ignoring the possibility that this user was present in /etc/groups.
-						LOG.warn("Group Name could not be found for group id: [" + groupId + "]. Skipping adding
user [" + userName + "] with id [" + userId + "].") ;
-					}
+
+				// if a user comes from an external group we might not have a uid
+				if (numUserId < minimumUserId && numUserId != -1)
+					continue;
+
+
+				// "id" is same across Linux / BSD / MacOSX
+				// gids are used as id might return groups with spaces, ie "domain users"
+				Process process = Runtime.getRuntime().exec(
+						new String[]{"bash", "-c", "id -G " + entry.getKey()});
+
+				reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+				line = reader.readLine();
+				reader.close();
+
+				LOG.debug("id -G returned " + line);
+
+				if (line.trim().isEmpty()) {
+					LOG.warn("User " + entry.getKey() + " could not be resolved");
+					continue;
 				}
-				else {
-					LOG.debug("Skipping user [" + userName + "] since its userid [" + userId + "] is less
than minuserid limit [" + minimumUserId + "].");
+
+				String[] gids = line.split(" ");
+
+				// check if all groups returned by id are visible to ranger
+				ArrayList<String> allowedGroups = new ArrayList<String>();
+				for (String gid : gids) {
+					int numGroupId = Integer.parseInt(gid);
+					if (numGroupId < minimumGroupId)
+						continue;
+
+					String groupName = groupId2groupNameMap.get(gid);
+					if (groupName != null)
+						allowedGroups.add(groupName);
 				}
-			}
-			
-			reader.close() ;
-			
-			passwordFileModiiedAt = f.lastModified() ;
 
+				user2GroupListMap.put(entry.getKey(), allowedGroups);
+			}
+			LOG.debug("End drill down group members");
 		}
 	}
 
-	
-	private void buildUnixGroupList() throws Throwable {
-		
-		File f = new File(UNIX_GROUP_FILE) ;
-		
-		if (f.exists()) {
-			
-			BufferedReader reader = null ;
-			
-			reader = new BufferedReader(new FileReader(f)) ;
-			
-			String line = null ;
-			
-			
-			
-			while ( (line = reader.readLine()) != null) {
-
-				if (line.trim().isEmpty()) 
-					continue ;
-				
-				String[] tokens = line.split(":") ;
-				
-				int len = tokens.length ;
-				
-				if (len < 2) {
-					continue ;
-				}
-				
-				String groupName = tokens[0] ;
-				String groupId = tokens[2] ;
-				String groupMembers = null ;
-				
-				if (tokens.length > 3) {
-					groupMembers = tokens[3] ;
+	private void parseMembers(String line) {
+		if (line == null || line.isEmpty())
+			return;
+
+		String[] tokens = line.split(":");
+
+		if (tokens.length < 2)
+			return;
+
+		String groupName = tokens[0];
+		String groupId = tokens[2];
+		String groupMembers = null;
+
+		if (tokens.length > 3)
+			groupMembers = tokens[3];
+
+		if (groupId2groupNameMap.containsKey(groupId)) {
+			groupId2groupNameMap.remove(groupId);
+		}
+
+		int numGroupId = Integer.parseInt(groupId);
+		if (numGroupId < minimumGroupId)
+			return;
+
+		groupId2groupNameMap.put(groupId, groupName);
+
+		if (groupMembers != null && !groupMembers.trim().isEmpty()) {
+			for (String user : groupMembers.split(",")) {
+				List<String> groupList = internalUser2GroupListMap.get(user);
+				if (groupList == null) {
+					groupList = new ArrayList<String>();
+					internalUser2GroupListMap.put(user, groupList);
 				}
-				
-				if (groupId2groupNameMap.containsKey(groupId)) {
-					groupId2groupNameMap.remove(groupId) ;
+				if (!groupList.contains(groupName)) {
+					groupList.add(groupName);
 				}
-				
-				groupId2groupNameMap.put(groupId,groupName) ;
-				// also build an internal map of users to their group list which is consulted by user
list creator
-				if (groupMembers != null && ! groupMembers.trim().isEmpty()) {
-					for(String user : groupMembers.split(",")) {
-						List<String> groupList = internalUser2GroupListMap.get(user) ;
-						if (groupList == null) {
-							groupList = new ArrayList<String>() ;
-							internalUser2GroupListMap.put(user, groupList) ;
-						}
-						if (! groupList.contains(groupName)) {
-							groupList.add(groupName) ;
-						}
-					}
+			}
+		}
+	}
+
+	private void buildUnixGroupList(String allGroupsCmd, String groupCmd, boolean useGid) throws
Throwable {
+		LOG.debug("Start enumerating groups");
+		BufferedReader reader;
+
+		if (!useNss) {
+			File file = new File(UNIX_GROUP_FILE);
+			groupFileModifiedAt = file.lastModified();
+			reader = new BufferedReader(new FileReader(file)) ;
+		} else {
+			Process process = Runtime.getRuntime().exec(
+					new String[]{"bash", "-c", allGroupsCmd});
+			reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+		}
+
+		String line = null;
+
+		while ((line = reader.readLine()) != null) {
+			if (line.trim().isEmpty())
+				continue;
+
+			parseMembers(line);
+		}
+
+		reader.close();
+
+		LOG.debug("End enumerating group");
+
+		if (!useNss)
+			return;
+
+		if (enumerateGroupMembers) {
+			LOG.debug("Start enumerating group members");
+			Map<String,String> copy = new HashMap<String, String>(groupId2groupNameMap);
+
+			for (Map.Entry<String, String> group : copy.entrySet()) {
+				LOG.debug("Enumerating group: " + group.getValue() + " GID(" + group.getKey() + ")");
+
+				String command;
+				if (useGid) {
+					command = String.format(groupCmd, group.getKey());
+				} else {
+					command = String.format(groupCmd, group.getValue());
 				}
 
-				
+				String[] cmd = new String[]{"bash", "-c", command + " " + group.getKey()};
+				LOG.debug("Executing: " + Arrays.toString(cmd));
+
+				Process process = Runtime.getRuntime().exec(cmd);
+				reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+				line = reader.readLine();
+				reader.close();
+
+				LOG.debug("bash -c " + command + " for group " + group + " returned " + line);
+
+				parseMembers(line);
 			}
-			
-			reader.close() ;
-			
-			groupFileModifiedAt = f.lastModified() ;
+			LOG.debug("End enumerating group members");
+		}
+
+		if (config.getEnumerateGroups() != null) {
+			String[] groups = config.getEnumerateGroups().split(",");
+
+			LOG.debug("Adding extra groups");
+
+			for (String group : groups) {
+				String command = groupCmd.format(group);
+				String[] cmd = new String[]{"bash", "-c", command + " '" + group + "'"};
+				LOG.debug("Executing: " + Arrays.toString(cmd));
+
+				Process process = Runtime.getRuntime().exec(cmd);
+				reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+				line = reader.readLine();
+				reader.close();
+				LOG.debug("bash -c " + command + " for group " + group + " returned " + line);
 
-		
+				parseMembers(line);
+			}
+			LOG.debug("Done adding extra groups");
 		}
 	}
-	
+
+	@VisibleForTesting
+	Map<String,List<String>> getUser2GroupListMap() {
+		return user2GroupListMap;
+	}
+
+	@VisibleForTesting
+	Map<String,String> getGroupId2groupNameMap() {
+		return groupId2groupNameMap;
+	}
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/658f2310/ugsync/src/test/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilderTest.java
----------------------------------------------------------------------
diff --git a/ugsync/src/test/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilderTest.java
b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilderTest.java
new file mode 100644
index 0000000..e4d5456
--- /dev/null
+++ b/ugsync/src/test/java/org/apache/ranger/unixusersync/process/UnixUserGroupBuilderTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.unixusersync.process;
+
+import org.apache.ranger.unixusersync.config.UserGroupSyncConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+import static org.hamcrest.Matchers.*;
+
+public class UnixUserGroupBuilderTest {
+    private UserGroupSyncConfig config;
+
+    @Before
+    public void setUp() throws Exception {
+        config = UserGroupSyncConfig.getInstance();
+        config.setProperty("ranger.usersync.unix.minUserId", "0");
+        config.setProperty("ranger.usersync.unix.minGroupId", "0");
+    }
+
+    @Test
+    public void testBuilderPasswd() throws Throwable {
+        config.setProperty("ranger.usersync.unix.backend", "passwd");
+
+        UnixUserGroupBuilder builder = new UnixUserGroupBuilder();
+        builder.init();
+
+        Map<String, String> groups = builder.getGroupId2groupNameMap();
+        String name = groups.get("0");
+        assertThat(name, anyOf(equalTo("wheel"), equalTo("root")));
+
+        Map<String, List<String>> users = builder.getUser2GroupListMap();
+        List<String> usergroups = users.get("root");
+        assertNotNull(usergroups);
+        assertThat(usergroups, anyOf(hasItem("wheel"), hasItem("root")));
+
+    }
+
+    @Test
+    public void testBuilderNss() throws Throwable {
+        config.setProperty("ranger.usersync.unix.backend", "nss");
+
+        UnixUserGroupBuilder builder = new UnixUserGroupBuilder();
+        builder.init();
+
+        Map<String, String> groups = builder.getGroupId2groupNameMap();
+        String name = groups.get("0");
+        assertThat(name, anyOf(equalTo("wheel"), equalTo("root")));
+
+        Map<String, List<String>> users = builder.getUser2GroupListMap();
+        List<String> usergroups = users.get("root");
+        assertNotNull(usergroups);
+        assertThat(usergroups, anyOf(hasItem("wheel"), hasItem("root")));
+    }
+
+    @Test
+    public void testBuilderExtraGroups() throws Throwable {
+        config.setProperty("ranger.usersync.unix.backend", "nss");
+        config.setProperty("ranger.usersync.group.enumerategroup", "root,wheel,daemon");
+
+        UnixUserGroupBuilder builder = new UnixUserGroupBuilder();
+        builder.init();
+
+        // this is not a full test as it cannot be mocked sufficiently
+        Map<String, String> groups = builder.getGroupId2groupNameMap();
+        assertTrue(groups.containsValue("daemon"));
+        assertThat(groups, anyOf(hasValue("wheel"), hasValue("root")));
+    }
+
+    @Test
+    public void testMinUidGid() throws Throwable {
+        config.setProperty("ranger.usersync.unix.backend", "nss");
+        config.setProperty("ranger.usersync.unix.minUserId", "500");
+        config.setProperty("ranger.usersync.unix.minGroupId", "500");
+
+        UnixUserGroupBuilder builder = new UnixUserGroupBuilder();
+        builder.init();
+
+        // this is not a full test as it cannot be mocked sufficiently
+        Map<String, String> groups = builder.getGroupId2groupNameMap();
+        assertFalse(groups.containsValue("wheel"));
+
+        Map<String, List<String>> users = builder.getUser2GroupListMap();
+        assertNull(users.get("root"));
+    }
+
+}


Mime
View raw message