ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject incubator-ranger git commit: RANGER-1032: Update TagSync installation to support kerberized mode configurations
Date Tue, 14 Jun 2016 13:50:33 GMT
Repository: incubator-ranger
Updated Branches:
  refs/heads/master 97deb2b6f -> ce45eff83


RANGER-1032: Update TagSync installation to support kerberized mode configurations

Signed-off-by: Madhan Neethiraj <madhan@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/ce45eff8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/ce45eff8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/ce45eff8

Branch: refs/heads/master
Commit: ce45eff839544371c80f68bddf2ef47573cd210c
Parents: 97deb2b
Author: Abhay Kulkarni <akulkarni@hortonworks.com>
Authored: Thu May 12 17:11:36 2016 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Tue Jun 14 06:20:02 2016 -0700

----------------------------------------------------------------------
 src/main/assembly/tagsync.xml                   |  3 -
 .../conf/templates/installprop2xml.properties   | 13 +++
 .../conf/templates/ranger-tagsync-template.xml  | 16 ++++
 tagsync/scripts/install.properties              | 21 ++++-
 tagsync/scripts/ranger-tagsync-upload.sh        |  2 +-
 tagsync/scripts/setup.py                        | 87 +++++++++++++++----
 tagsync/scripts/updatetagadminpassword.py       | 61 ++++++++++----
 .../ranger/tagsync/process/TagSyncConfig.java   | 89 ++++++++++++++++----
 .../source/atlasrest/AtlasRESTTagSource.java    | 58 +++++++++++--
 .../tagsync/source/atlasrest/AtlasRESTUtil.java | 81 +++++++++++++++---
 .../src/main/resources/ranger-tagsync-site.xml  | 18 ++--
 11 files changed, 362 insertions(+), 87 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/src/main/assembly/tagsync.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml
index fbcca8f..bb5b35c 100644
--- a/src/main/assembly/tagsync.xml
+++ b/src/main/assembly/tagsync.xml
@@ -93,9 +93,6 @@
 	    	<fileMode>644</fileMode>
 		<outputDirectory>/conf.dist</outputDirectory>
 		<directory>tagsync/conf.dist</directory>
-		<excludes>
-			<exclude>etc</exclude>
-		</excludes>
 	</fileSet>
 	<fileSet>
 	    	<directoryMode>755</directoryMode>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/conf/templates/installprop2xml.properties
----------------------------------------------------------------------
diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties
index aa0c568..510cd6f 100644
--- a/tagsync/conf/templates/installprop2xml.properties
+++ b/tagsync/conf/templates/installprop2xml.properties
@@ -26,11 +26,20 @@ TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = atlas.kafka.bootstrap.servers
 TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = atlas.kafka.zookeeper.connect
 TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = atlas.kafka.entities.group.id
 
+TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME = atlas.kafka.sasl.kerberos.service.name
+TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL = atlas.kafka.security.protocol
+
+TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL = atlas.jaas.kafkaClient.option.principal
+TAG_SOURCE_ATLAS_KERBEROS_KEYTAB = atlas.jaas.kafkaClient.option.keyTab
+
 TAG_SOURCE_ATLASREST_ENABLED = ranger.tagsync.source.atlasrest
 
 TAG_SOURCE_ATLASREST_ENDPOINT = ranger.tagsync.source.atlasrest.endpoint
 TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = ranger.tagsync.source.atlasrest.download.interval.millis
 
+TAG_SOURCE_ATLASREST_USERNAME = ranger.tagsync.source.atlasrest.username
+TAG_SOURCE_ATLASREST_PASSWORD = ranger.tagsync.source.atlasrest.password
+
 TAG_SOURCE_FILE_ENABLED = ranger.tagsync.source.file
 
 TAG_SOURCE_FILE_FILENAME = ranger.tagsync.source.file.filename
@@ -40,6 +49,9 @@ TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.ranger.service
 TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.atlas.custom.resource.mappers
 
 TAGSYNC_KEYSTORE_FILENAME = ranger.tagsync.keystore.filename
+TAG_SOURCE_ATLASREST_KEYSTORE_FILENAME = ranger.tagsync.source.atlasrest.keystore.filename
+TAG_SOURCE_ATLASREST_SSL_CONFIG_FILENAME = ranger.tagsync.source.atlasrest.ssl.config.filename
+
 
 unix_user   = unix_user
 unix_group  = unix_group
@@ -49,6 +61,7 @@ logdir = ranger.tagsync.logdir
 tagsync_principal = ranger.tagsync.kerberos.principal
 tagsync_keytab = ranger.tagsync.kerberos.keytab
 
+
 # TODO - What property in ranger-tagsync-site.xml should hadoop_conf map to??
 hadoop_conf = hadoop_conf
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/conf/templates/ranger-tagsync-template.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml
index 554a4e0..41aacbf 100644
--- a/tagsync/conf/templates/ranger-tagsync-template.xml
+++ b/tagsync/conf/templates/ranger-tagsync-template.xml
@@ -87,4 +87,20 @@
 		<name>ranger.tagsync.dest.ranger.username</name>
 		<value></value>
 	</property>
+	<property>
+		<name>ranger.tagsync.source.atlasrest.username</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.atlasrest.password</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.atlasrest.keystore.filename</name>
+		<value></value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.atlasrest.ssl.config.filename</name>
+		<value></value>
+	</property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/scripts/install.properties
----------------------------------------------------------------------
diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties
index 90878d8..add1a26 100644
--- a/tagsync/scripts/install.properties
+++ b/tagsync/scripts/install.properties
@@ -32,18 +32,25 @@ TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = localhost:6667
 TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = localhost:2181
 TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = ranger_entities_consumer
 
+TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME = kafka
+TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL = SASL_PLAINTEXT
+
+TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL =
+TAG_SOURCE_ATLAS_KERBEROS_KEYTAB =
+
 TAG_SOURCE_ATLASREST_ENABLED = false
 
 TAG_SOURCE_ATLASREST_ENDPOINT = http://localhost:21000
 TAG_SOURCE_ATLASREST_DOWNLOAD_INTERVAL_IN_MILLIS = 900000
 
+TAG_SOURCE_ATLASREST_USERNAME =
+TAG_SOURCE_ATLASREST_PASSWORD =
+
 TAG_SOURCE_FILE_ENABLED = false
 
 TAG_SOURCE_FILE_FILENAME = /etc/ranger/data/tags.json
 TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = 60000
 
-TAG_SOURCE_NONE_ENABLED = false
-
 # Mapping from Atlas hive cluster-name to Ranger service-name
 # this needs to be in format clusterName,componentType,serviceName;clusterName2,componentType2,serviceName2
 # Note that there are no blanks anywhere in the value-string
@@ -74,6 +81,13 @@ TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS=
 #
 TAGSYNC_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/rangertagsync.jceks
 
+# File where Atlas credentials is kept in cryptic format
+
+TAG_SOURCE_ATLASREST_KEYSTORE_FILENAME = /etc/ranger/tagsync/conf/atlasuser.jceks
+
+# SSL config file name for HTTPS messages to tag source - Atlas-REST
+TAG_SOURCE_ATLASREST_SSL_CONFIG_FILENAME =
+
 #User and group for the tagsync process
 unix_user=ranger
 unix_group=ranger
@@ -82,7 +96,10 @@ unix_group=ranger
 logdir = log
 
 #Set to run in kerberos environment
+is_secure = false
 tagsync_principal=
 tagsync_keytab=
 
+
+
 hadoop_conf=/etc/hadoop/conf

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/scripts/ranger-tagsync-upload.sh
----------------------------------------------------------------------
diff --git a/tagsync/scripts/ranger-tagsync-upload.sh b/tagsync/scripts/ranger-tagsync-upload.sh
index 99f8dad..495903e 100755
--- a/tagsync/scripts/ranger-tagsync-upload.sh
+++ b/tagsync/scripts/ranger-tagsync-upload.sh
@@ -66,5 +66,5 @@ cp="${cdir}/conf:${cdir}/dist/*:${cdir}/lib/*"
 cd ${cdir}
 umask 0077
 
-java -Dproc_rangertagsync-${action} ${JAVA_OPTS} -Dlogdir="${logdir}" -cp "${cp}" ${className}
$* > ${logdir}/tagsync.out 2>&1
+java -Dproc_rangertagsync-${action} ${JAVA_OPTS} -Dlogdir="${logdir}" -Dlog4j.configuration=file:/etc/ranger/tagsync/conf/log4j.properties
-cp "${cp}" ${className} $* > ${logdir}/tagsync.out 2>&1
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/scripts/setup.py
----------------------------------------------------------------------
diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py
index ec35a10..ea47de7 100755
--- a/tagsync/scripts/setup.py
+++ b/tagsync/scripts/setup.py
@@ -71,7 +71,12 @@ initPrefixList = ['S99', 'K00']
 TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS'
 TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT'
 TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID'
-TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = 'ranger.tagsync.atlas.to.service.mapping'
+
+TAG_SOURCE_ATLAS_KAKFA_SERVICE_NAME_KEY = 'TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME'
+TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL_KEY = 'TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL'
+TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL_KEY = 'TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL'
+TAG_SOURCE_ATLAS_KERBEROS_KEYTAB_KEY = 'TAG_SOURCE_ATLAS_KERBEROS_KEYTAB'
+TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = 'ranger.tagsync.atlas.to.ranger.service.mapping'
 TAGSYNC_INSTALL_PROP_PREFIX_FOR_ATLAS_RANGER_MAPPING = 'ranger.tagsync.atlas.'
 TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = '.instance.'
 TAGSYNC_INSTALL_PROP_SUFFIX_FOR_ATLAS_RANGER_MAPPING = '.ranger.service'
@@ -88,6 +93,7 @@ TAG_SOURCE_FILE_ENABLED = 'ranger.tagsync.source.file'
 hadoopConfFileName = 'core-site.xml'
 ENV_HADOOP_CONF_FILE = "ranger-tagsync-env-hadoopconfdir.sh"
 globalDict = {}
+configure_security = False
 
 RANGER_TAGSYNC_HOME = os.getenv("RANGER_TAGSYNC_HOME")
 if RANGER_TAGSYNC_HOME is None:
@@ -209,6 +215,9 @@ def convertInstallPropsToXML(props):
 
 	atlasOutFile = file(atlasOutFn, "w")
 
+	atlas_principal = ''
+	atlas_keytab = ''
+
 	for k,v in props.iteritems():
 		if (k in directKeyMap.keys()):
 			newKey = directKeyMap[k]
@@ -218,10 +227,27 @@ def convertInstallPropsToXML(props):
 				atlasOutFile.write(newKey + "=" + v + "\n")
 			elif (k == TAGSYNC_ATLAS_CONSUMER_GROUP_KEY):
 				atlasOutFile.write(newKey + "=" + v + "\n")
+			elif (configure_security and k == TAG_SOURCE_ATLAS_KAKFA_SERVICE_NAME_KEY):
+				atlasOutFile.write(newKey + "=" + v + "\n")
+			elif (configure_security and k == TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL_KEY):
+				atlasOutFile.write(newKey + "=" + v + "\n")
+			elif (configure_security and k == TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL_KEY):
+				atlas_principal = v
+			elif (configure_security and k == TAG_SOURCE_ATLAS_KERBEROS_KEYTAB_KEY):
+				atlas_keytab = v
 			else:
 				ret[newKey] = v
 		else:
-			print "Direct Key not found:%s" % (k)
+			print "INFO: Direct Key not found:%s" % (k)
+
+	if (configure_security):
+		atlasOutFile.write("atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule"
+ "\n")
+		atlasOutFile.write("atlas.jaas.KafkaClient.loginModuleControlFlag = required" + "\n")
+		atlasOutFile.write("atlas.jaas.KafkaClient.option.useKeyTab = true" + "\n")
+		atlasOutFile.write("atlas.jaas.KafkaClient.option.storeKey = true" + "\n")
+		atlasOutFile.write("atlas.jaas.KafkaClient.option.serviceName = kafka" + "\n")
+		atlasOutFile.write("atlas.jaas.KafkaClient.option.keyTab = " + atlas_keytab + "\n")
+		atlasOutFile.write("atlas.jaas.KafkaClient.option.principal = " + atlas_principal + "\n")
 
 	atlasOutFile.close()
 
@@ -291,16 +317,27 @@ def initializeInitD():
 def write_env_files(exp_var_name, log_path, file_name):
         final_path = "{0}/{1}".format(confBaseDirName,file_name)
         if not os.path.isfile(final_path):
-                print "Creating %s file" % file_name
+                print "INFO: Creating %s file" % file_name
         f = open(final_path, "w")
         f.write("export {0}={1}".format(exp_var_name,log_path))
         f.close()
 
 def main():
 
+	global configure_security
+
 	print "\nINFO: Installing ranger-tagsync .....\n"
 
 	populate_global_dict()
+
+
+	kerberize = globalDict['is_secure']
+	if kerberize != "":
+		kerberize = kerberize.lower()
+		if kerberize == "true" or kerberize == "enabled" or kerberize == "yes":
+			configure_security = True
+
+
 	hadoop_conf = globalDict['hadoop_conf']
 
 	dirList = [ rangerBaseDirName, tagsyncBaseDirFullName, confFolderName ]
@@ -393,15 +430,28 @@ def main():
 	if ('ranger.tagsync.dest.ranger.username' not in mergeProps):
 		mergeProps['ranger.tagsync.dest.ranger.username'] = 'rangertagsync'
 
-	if (tagsyncKSPath == ''):
-		mergeProps['ranger.tagsync.dest.ranger.password'] = 'rangertagsync'
-
-	else:
+	if (tagsyncKSPath != ''):
 		tagadminPasswd = 'rangertagsync'
 		tagadminAlias = 'tagadmin.user.password'
 		updatePropertyInJCKSFile(tagsyncKSPath,tagadminAlias,tagadminPasswd)
 		os.chown(tagsyncKSPath,ownerId,groupId)
 
+	tagsyncAtlasKSPath = mergeProps['ranger.tagsync.source.atlasrest.keystore.filename']
+
+	if ('ranger.tagsync.source.atlasrest.username' not in mergeProps):
+		mergeProps['ranger.tagsync.source.atlasrest.username'] = 'admin'
+
+	if (tagsyncAtlasKSPath != ''):
+		if ('ranger.tagsync.source.atlasrest.password' not in mergeProps):
+			atlasPasswd = 'admin'
+		else:
+			atlasPasswd = mergeProps['ranger.tagsync.source.atlasrest.password']
+			mergeProps.pop('ranger.tagsync.source.atlasrest.password')
+
+		atlasAlias = 'atlas.user.password'
+		updatePropertyInJCKSFile(tagsyncAtlasKSPath,atlasAlias,atlasPasswd)
+		os.chown(tagsyncAtlasKSPath,ownerId,groupId)
+
 	writeXMLUsingProperties(fn, mergeProps, outfn)
 
 	fixPermList = [ ".", tagsyncBaseDirName, confFolderName ]
@@ -419,25 +469,26 @@ def main():
 				os.chown(fn, ownerId, groupId)
 				os.chmod(fn, 0755)
 
-	write_env_files("RANGER_TAGSYNC_HADOOP_CONF_DIR", hadoop_conf, ENV_HADOOP_CONF_FILE);
+	write_env_files("RANGER_TAGSYNC_HADOOP_CONF_DIR", hadoop_conf, ENV_HADOOP_CONF_FILE)
 	os.chown(os.path.join(confBaseDirName, ENV_HADOOP_CONF_FILE),ownerId,groupId)
-        os.chmod(os.path.join(confBaseDirName, ENV_HADOOP_CONF_FILE),0755)
+	os.chmod(os.path.join(confBaseDirName, ENV_HADOOP_CONF_FILE),0755)
 
 	hadoop_conf_full_path = os.path.join(hadoop_conf, hadoopConfFileName)
-        tagsync_conf_full_path = os.path.join(tagsyncBaseDirFullName,confBaseDirName,hadoopConfFileName)
-        if not isfile(hadoop_conf_full_path):
-                print "WARN: core-site.xml file not found in provided hadoop conf path..."
+	tagsync_conf_full_path = os.path.join(tagsyncBaseDirFullName,confBaseDirName,hadoopConfFileName)
+
+	if not isfile(hadoop_conf_full_path):
+		print "WARN: core-site.xml file not found in provided hadoop conf path..."
 		f = open(tagsync_conf_full_path, "w")
-                f.write("<configuration></configuration>")
-                f.close()
+		f.write("<configuration></configuration>")
+		f.close()
 		os.chown(tagsync_conf_full_path,ownerId,groupId)
 		os.chmod(tagsync_conf_full_path,0750)
 	else:
-	        if os.path.islink(tagsync_conf_full_path):
-        	        os.remove(tagsync_conf_full_path)
+		if os.path.islink(tagsync_conf_full_path):
+			os.remove(tagsync_conf_full_path)
 
-	if isfile(hadoop_conf_full_path):
-	        os.symlink(hadoop_conf_full_path, tagsync_conf_full_path)
+	if isfile(hadoop_conf_full_path) and not isfile(tagsync_conf_full_path):
+			os.symlink(hadoop_conf_full_path, tagsync_conf_full_path)
 
 	print "\nINFO: Completed ranger-tagsync installation.....\n"
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/scripts/updatetagadminpassword.py
----------------------------------------------------------------------
diff --git a/tagsync/scripts/updatetagadminpassword.py b/tagsync/scripts/updatetagadminpassword.py
index d2efef5..0b1f2ca 100644
--- a/tagsync/scripts/updatetagadminpassword.py
+++ b/tagsync/scripts/updatetagadminpassword.py
@@ -109,32 +109,61 @@ def main():
 	log("[I] Using Java:" + str(JAVA_BIN),"info")
 
 	globalDict=import_properties_from_xml(CFG_FILE,globalDict)
-	TAGSYNC_KEYSTORE_FILENAME=globalDict['ranger.tagsync.keystore.filename']
-	log("[I] TAGSYNC_KEYSTORE_FILENAME:" + str(TAGSYNC_KEYSTORE_FILENAME),"info")
-	TAGSYNC_TAGADMIN_ALIAS="tagadmin.user.password"
-	TAGSYNC_TAGADMIN_PASSWORD = ''
-	TAGSYNC_TAGADMIN_USERNAME = ''
+
+	ENDPOINT=''
+	KEYSTORE_FILENAME=''
+	KEYSTORE_FILENAME_PROMPT=''
+	ALIAS = ''
+	USERNAME=''
+	PASSWORD=''
+	USERNAME_PROPERTY_NAME=''
+	FILENAME_PROPERTY_NAME=''
+
+	while ENDPOINT == "" or not (ENDPOINT == "ATLAS" or ENDPOINT == "RANGER"):
+		print "Enter Destination NAME (Ranger/Atlas):"
+		ENDPOINT=raw_input()
+		ENDPOINT = ENDPOINT.upper()
+
+	if ENDPOINT == "ATLAS":
+		USERNAME_PROPERTY_NAME='ranger.tagsync.source.atlasrest.username'
+		FILENAME_PROPERTY_NAME='ranger.tagsync.source.atlasrest.keystore.filename'
+		ALIAS="atlas.user.password"
+		KEYSTORE_FILENAME_PROMPT='RANGER_TAGSYNC_ATLAS_KEYSTORE_FILENAME'
+
+	elif ENDPOINT == "RANGER":
+		USERNAME_PROPERTY_NAME='ranger.tagsync.dest.ranger.username'
+		FILENAME_PROPERTY_NAME='ranger.tagsync.keystore.filename'
+		ALIAS="tagadmin.user.password"
+		KEYSTORE_FILENAME_PROMPT='RANGER_TAGSYNC_RANGER_KEYSTORE_FILENAME'
+
+	else:
+		log("[E] Unsupported ENDPOINT[" + ENDPOINT + "]")
+		return
+
+	KEYSTORE_FILENAME = globalDict[FILENAME_PROPERTY_NAME]
+
+	log("[I] " + KEYSTORE_FILENAME_PROMPT + ":" + str(KEYSTORE_FILENAME),"info")
 	unix_user = "ranger"
 	unix_group = "ranger"
 
-	while TAGSYNC_TAGADMIN_USERNAME == "":
-		print "Enter tagadmin user name:"
-		TAGSYNC_TAGADMIN_USERNAME=raw_input()
+	while USERNAME == "":
+		print "Enter " + ENDPOINT + " user name:"
+		USERNAME=raw_input()
 
-	while TAGSYNC_TAGADMIN_PASSWORD == "":
-		TAGSYNC_TAGADMIN_PASSWORD=getpass.getpass("Enter tagadmin user password:")
+	while PASSWORD == "":
+		PASSWORD=getpass.getpass("Enter " + ENDPOINT + " user password:")
 
-	if TAGSYNC_KEYSTORE_FILENAME != "" or TAGSYNC_TAGADMIN_USERNAME != "" or TAGSYNC_TAGADMIN_PASSWORD
!= "":
-		log("[I] Storing tagadmin tagsync password in credential store:","info")
-		cmd="%s -cp lib/* org.apache.ranger.credentialapi.buildks create %s -value %s  -provider
jceks://file%s" %(JAVA_BIN,TAGSYNC_TAGADMIN_ALIAS,TAGSYNC_TAGADMIN_PASSWORD,TAGSYNC_KEYSTORE_FILENAME)
+	if KEYSTORE_FILENAME != "" or USERNAME != "" or PASSWORD != "":
+		log("[I] Storing " + ENDPOINT + " tagsync password in credential store:","info")
+		cmd="%s -cp lib/* org.apache.ranger.credentialapi.buildks create %s -value %s  -provider
jceks://file%s" %(JAVA_BIN,ALIAS,PASSWORD,KEYSTORE_FILENAME)
 		ret=subprocess.call(shlex.split(cmd))
 		if ret == 0:
-			cmd="chown %s:%s %s" %(unix_user,unix_group,TAGSYNC_KEYSTORE_FILENAME)
+			cmd="chown %s:%s %s" %(unix_user,unix_group,KEYSTORE_FILENAME)
 			ret=subprocess.call(shlex.split(cmd))
 			if ret == 0:
 				if os.path.isfile(CFG_FILE):
-					write_properties_to_xml(CFG_FILE,"ranger.tagsync.dest.ranger.username",TAGSYNC_TAGADMIN_USERNAME)
-					write_properties_to_xml(CFG_FILE,"ranger.tagsync.keystore.filename",TAGSYNC_KEYSTORE_FILENAME)
+					write_properties_to_xml(CFG_FILE,USERNAME_PROPERTY_NAME,USERNAME)
+					write_properties_to_xml(CFG_FILE,FILENAME_PROPERTY_NAME,KEYSTORE_FILENAME)
 				else:
 					log("[E] Required file not found: ["+CFG_FILE+"]","error")
 			else:

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
index 4cb8dea..c52e0d2 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
@@ -24,7 +24,11 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.SecureClientLogin;
 import org.apache.log4j.Logger;
 
-import java.io.*;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.util.Enumeration;
@@ -52,10 +56,13 @@ public class TagSyncConfig extends Configuration {
 	private static final String TAGSYNC_SINK_CLASS_PROP = "ranger.tagsync.dest.ranger.impl.class";
 
 	private static final String TAGSYNC_DEST_RANGER_PASSWORD_ALIAS = "tagadmin.user.password";
+	private static final String TAGSYNC_SOURCE_ATLASREST_PASSWORD_ALIAS = "atlas.user.password";
 
 	private static final String TAGSYNC_TAGADMIN_USERNAME_PROP = "ranger.tagsync.dest.ranger.username";
+	private static final String TAGSYNC_ATLASREST_USERNAME_PROP = "ranger.tagsync.source.atlasrest.username";
 
 	private static final String TAGSYNC_TAGADMIN_PASSWORD_PROP = "ranger.tagsync.dest.ranger.password";
+	private static final String TAGSYNC_ATLASREST_PASSWORD_PROP = "ranger.tagsync.source.atlasrest.password";
 
 	private static final String TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL_PROP = "ranger.tagsync.dest.ranger.connection.check.interval";
 
@@ -65,17 +72,22 @@ public class TagSyncConfig extends Configuration {
 
 	private static final String TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP = "ranger.tagsync.source.atlasrest.download.interval.millis";
 
+	private static final String TAGSYNC_ATLAS_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.source.atlasrest.ssl.config.filename";
+
 	public static final String TAGSYNC_FILESOURCE_FILENAME_PROP = "ranger.tagsync.source.file.filename";
 
 	private static final String TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL_PROP = "ranger.tagsync.source.file.check.interval.millis";
 
 	private static final String TAGSYNC_TAGADMIN_KEYSTORE_PROP = "ranger.tagsync.keystore.filename";
+	private static final String TAGSYNC_ATLASREST_KEYSTORE_PROP = "ranger.tagsync.source.atlasrest.keystore.filename";
 
 	private static final String DEFAULT_TAGADMIN_USERNAME = "rangertagsync";
 	private static final String DEFAULT_TAGADMIN_PASSWORD = "rangertagsync";
+	private static final String DEFAULT_ATLASREST_USERNAME = "admin";
+	private static final String DEFAULT_ATLASREST_PASSWORD = "admin";
 
 	private static final int DEFAULT_TAGSYNC_TAGADMIN_CONNECTION_CHECK_INTERVAL = 15000;
-	private static final long DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL = 900000;
+	private static final long DEFAULT_TAGSYNC_ATLASREST_SOURCE_DOWNLOAD_INTERVAL = 900000;
 	private static final long DEFAULT_TAGSYNC_FILESOURCE_MOD_TIME_CHECK_INTERVAL = 60000;
 
 	private static final String AUTH_TYPE = "hadoop.security.authentication";
@@ -210,7 +222,7 @@ public class TagSyncConfig extends Configuration {
 
 	static public long getTagSourceAtlasDownloadIntervalInMillis(Properties prop) {
 		String val = prop.getProperty(TAGSYNC_ATLAS_REST_SOURCE_DOWNLOAD_INTERVAL_PROP);
-		long ret = DEFAULT_TAGSYNC_REST_SOURCE_DOWNLOAD_INTERVAL;
+		long ret = DEFAULT_TAGSYNC_ATLASREST_SOURCE_DOWNLOAD_INTERVAL;
 		if (StringUtils.isNotBlank(val)) {
 			try {
 				ret = Long.valueOf(val);
@@ -233,18 +245,6 @@ public class TagSyncConfig extends Configuration {
 		return prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP);
 	}
 
-	static public String getTagAdminRESTSslConfigFile(Properties prop) {
-		return prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP);
-	}
-
-	static public String getTagSourceFileName(Properties prop) {
-		return prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP);
-	}
-
-	static public String getAtlasEndpoint(Properties prop) {
-		return prop.getProperty(TAGSYNC_ATLASSOURCE_ENDPOINT_PROP);
-	}
-
 	static public String getTagAdminPassword(Properties prop) {
 		//update credential from keystore
 		String password = null;
@@ -286,8 +286,61 @@ public class TagSyncConfig extends Configuration {
 		return userName;
 	}
 
-	static public String getAtlasSslConfigFileName(Properties prop) {
-		return "";
+	static public String getTagAdminRESTSslConfigFile(Properties prop) {
+		return prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP);
+	}
+
+	static public String getTagSourceFileName(Properties prop) {
+		return prop.getProperty(TAGSYNC_FILESOURCE_FILENAME_PROP);
+	}
+
+	static public String getAtlasRESTEndpoint(Properties prop) {
+		return prop.getProperty(TAGSYNC_ATLASSOURCE_ENDPOINT_PROP);
+	}
+
+	static public String getAtlasRESTPassword(Properties prop) {
+		//update credential from keystore
+		String password = null;
+		if (prop != null && prop.containsKey(TAGSYNC_ATLASREST_PASSWORD_PROP)) {
+			password = prop.getProperty(TAGSYNC_ATLASREST_PASSWORD_PROP);
+			if (password != null && !password.isEmpty()) {
+				return password;
+			}
+		}
+		if (prop != null && prop.containsKey(TAGSYNC_ATLASREST_KEYSTORE_PROP)) {
+			String path = prop.getProperty(TAGSYNC_ATLASREST_KEYSTORE_PROP);
+			if (path != null) {
+				if (!path.trim().isEmpty()) {
+					try {
+						password = CredentialReader.getDecryptedString(path.trim(), TAGSYNC_SOURCE_ATLASREST_PASSWORD_ALIAS);
+					} catch (Exception ex) {
+						password = null;
+					}
+					if (password != null && !password.trim().isEmpty() && !password.trim().equalsIgnoreCase("none"))
{
+						return password;
+					}
+				}
+			}
+		}
+		if(StringUtils.isBlank(password)){
+			return DEFAULT_ATLASREST_PASSWORD;
+		}
+		return null;
+	}
+
+	static public String getAtlasRESTUserName(Properties prop) {
+		String userName=null;
+		if(prop!=null && prop.containsKey(TAGSYNC_ATLASREST_USERNAME_PROP)){
+			userName=prop.getProperty(TAGSYNC_ATLASREST_USERNAME_PROP);
+		}
+		if(StringUtils.isBlank(userName)){
+			userName=DEFAULT_ATLASREST_USERNAME;
+		}
+		return userName;
+	}
+
+	static public String getAtlasRESTSslConfigFile(Properties prop) {
+		return prop.getProperty(TAGSYNC_ATLAS_REST_SSL_CONFIG_FILE_PROP);
 	}
 
 	static public String getCustomAtlasResourceMappers(Properties prop) {
@@ -339,7 +392,7 @@ public class TagSyncConfig extends Configuration {
 
 		readConfigFile(CORE_SITE_FILE);
 		readConfigFile(DEFAULT_CONFIG_FILE);
-		readConfigFile(CONFIG_FILE);		
+		readConfigFile(CONFIG_FILE);
 
 		props = getProps();
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
index 11ca2d8..3f29de5 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
@@ -29,6 +29,8 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.ranger.plugin.util.RangerRESTClient;
 import org.apache.ranger.tagsync.model.AbstractTagSource;
 import org.apache.ranger.plugin.util.ServiceTags;
 import org.apache.ranger.tagsync.model.TagSink;
@@ -38,14 +40,24 @@ import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits;
 import org.apache.ranger.tagsync.source.atlas.AtlasNotificationMapper;
 import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil;
 
-import java.util.*;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 
 public class AtlasRESTTagSource extends AbstractTagSource implements Runnable {
 	private static final Log LOG = LogFactory.getLog(AtlasRESTTagSource.class);
 
-	private String atlasEndpoint;
+	static final String AUTH_TYPE_KERBEROS = "kerberos";
+
 	private long sleepTimeBetweenCycleInMillis;
 
+	AtlasRESTUtil atlasRESTUtil = null;
+
+	private String authenticationType;
+	private String principal;
+	private String keytab;
+	private String nameRules;
+
 	private Thread myThread = null;
 
 	public static void main(String[] args) {
@@ -90,11 +102,45 @@ public class AtlasRESTTagSource extends AbstractTagSource implements
Runnable {
 
 		boolean ret = AtlasResourceMapperUtil.initializeAtlasResourceMappers(properties);
 
-		atlasEndpoint = TagSyncConfig.getAtlasEndpoint(properties);
 		sleepTimeBetweenCycleInMillis = TagSyncConfig.getTagSourceAtlasDownloadIntervalInMillis(properties);
 
-		if (StringUtils.isEmpty(atlasEndpoint)) {
-			LOG.info("No AtlasEndpoint specified, Initial download of Atlas-entities cannot be done.");
+		String restUrl       = TagSyncConfig.getAtlasRESTEndpoint(properties);
+		String sslConfigFile = TagSyncConfig.getAtlasRESTSslConfigFile(properties);
+		String userName = TagSyncConfig.getAtlasRESTUserName(properties);
+		String password = TagSyncConfig.getAtlasRESTPassword(properties);
+
+		authenticationType = TagSyncConfig.getAuthenticationType(properties);
+		nameRules = TagSyncConfig.getNameRules(properties);
+		principal = TagSyncConfig.getKerberosPrincipal(properties);
+		keytab = TagSyncConfig.getKerberosKeytab(properties);
+
+		final boolean kerberized = StringUtils.isNotEmpty(authenticationType)
+				&& authenticationType.trim().equalsIgnoreCase(AtlasRESTTagSource.AUTH_TYPE_KERBEROS)
+				&& SecureClientLogin.isKerberosCredentialExists(principal, keytab);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("restUrl=" + restUrl);
+			LOG.debug("sslConfigFile=" + sslConfigFile);
+			LOG.debug("userName=" + userName);
+			LOG.debug("authenticationType=" + authenticationType);
+			LOG.debug("principal=" + principal);
+			LOG.debug("keytab=" + keytab);
+			LOG.debug("nameRules=" + nameRules);
+			LOG.debug("kerberized=" + kerberized);
+		}
+
+		if (StringUtils.isNotEmpty(restUrl)) {
+			if (!restUrl.endsWith("/")) {
+				restUrl += "/";
+			}
+			RangerRESTClient atlasRESTClient = new RangerRESTClient(restUrl, sslConfigFile);
+
+			if (!kerberized) {
+				atlasRESTClient.setBasicAuthInfo(userName, password);
+			}
+			atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, kerberized, authenticationType, principal,
keytab, nameRules);
+		} else {
+			LOG.info("AtlasEndpoint not specified, Initial download of Atlas-entities cannot be done.");
 			ret = false;
 		}
 
@@ -148,8 +194,6 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable
{
 
 	public void synchUp() {
 
-		AtlasRESTUtil atlasRESTUtil = new AtlasRESTUtil(atlasEndpoint);
-
 		List<AtlasEntityWithTraits> atlasEntitiesWithTraits = atlasRESTUtil.getEntitiesWithTraits();
 
 		if (CollectionUtils.isNotEmpty(atlasEntitiesWithTraits)) {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
index d7f983e..01aaec2 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
@@ -28,13 +28,20 @@ import org.apache.atlas.typesystem.json.InstanceSerialization;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.security.SecureClientLogin;
 import org.apache.log4j.Logger;
 import org.apache.ranger.admin.client.datatype.RESTResponse;
 import org.apache.ranger.plugin.util.RangerRESTClient;
 import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits;
 import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil;
 
-import java.util.*;
+import javax.security.auth.Subject;
+import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 
 @SuppressWarnings("unchecked")
 public class AtlasRESTUtil {
@@ -58,23 +65,26 @@ public class AtlasRESTUtil {
 
 	private final Gson gson = new Gson();
 
-	private RangerRESTClient atlasRESTClient;
+	private final RangerRESTClient atlasRESTClient;
+	private final String principal;
+	private final String keytab;
+	private final String nameRules;
+	private final boolean kerberized;
 
-	public AtlasRESTUtil(String atlasEndpoint) {
+	public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean kerberized, String authenticationType,
String principal, String keytab, String nameRules) {
 		if (LOG.isDebugEnabled()) {
-			LOG.debug("==> AtlasRESTUtil(" + atlasEndpoint + ")");
+			LOG.debug("==> AtlasRESTUtil()");
 		}
 
-		if (!atlasEndpoint.endsWith("/")) {
-			atlasEndpoint += "/";
-		}
+		this.kerberized = kerberized;
 
-		// This uses RangerRESTClient to invoke REST APIs on Atlas. It will work only if scheme
of URL is http
-		atlasRESTClient = new RangerRESTClient();
-		atlasRESTClient.setUrl(atlasEndpoint);
+		this.atlasRESTClient = atlasRESTClient;
+		this.principal = principal;
+		this.keytab = keytab;
+		this.nameRules = nameRules;
 
 		if (LOG.isDebugEnabled()) {
-			LOG.debug("<== AtlasRESTUtil(" + atlasEndpoint + ")");
+			LOG.debug("<== AtlasRESTUtil()");
 		}
 	}
 
@@ -231,7 +241,7 @@ public class AtlasRESTUtil {
 		return ret;
 	}
 
-	private Map<String, Object> atlasAPI(String endpoint) {
+	private Map<String, Object> atlasAPI(final String endpoint) {
 
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("==> atlasAPI(" + endpoint + ")");
@@ -239,7 +249,49 @@ public class AtlasRESTUtil {
 		Map<String, Object> ret = new HashMap<String, Object>();
 
 		try {
-			WebResource webResource = atlasRESTClient.getResource(endpoint);
+			if (kerberized) {
+				LOG.debug("Using kerberos authentication");
+				Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules) ;
+				if(LOG.isDebugEnabled()) {
+					LOG.debug("Using Principal = "+ principal + ", keytab = "+keytab);
+				}
+				ret = Subject.doAs(sub, new PrivilegedAction<Map<String, Object>>() {
+					@Override
+					public Map<String, Object> run() {
+						try{
+							return executeAtlasAPI(endpoint);
+						}catch (Exception e) {
+							LOG.error("Atlas API failed with message : ", e);
+						}
+						return null;
+					}
+				});
+			} else {
+				LOG.debug("Using basic authentication");
+				ret = executeAtlasAPI(endpoint);
+			}
+		} catch (Exception exception) {
+			LOG.error("Exception when fetching Atlas objects.", exception);
+			ret = null;
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== atlasAPI(" + endpoint + ")");
+		}
+		return ret;
+	}
+
+	private Map<String, Object> executeAtlasAPI(final String endpoint) {
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> executeAtlasAPI(" + endpoint + ")");
+		}
+
+		Map<String, Object> ret = new HashMap<String, Object>();
+
+		try {
+			final WebResource webResource = atlasRESTClient.getResource(endpoint);
+
 			ClientResponse response = webResource.accept(REST_MIME_TYPE_JSON).type(REST_MIME_TYPE_JSON).get(ClientResponse.class);
 
 			if (response != null && response.getStatus() == 200) {
@@ -255,8 +307,9 @@ public class AtlasRESTUtil {
 		}
 
 		if (LOG.isDebugEnabled()) {
-			LOG.debug("<== atlasAPI(" + endpoint + ")");
+			LOG.debug("<== executeAtlasAPI(" + endpoint + ")");
 		}
+
 		return ret;
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ce45eff8/tagsync/src/main/resources/ranger-tagsync-site.xml
----------------------------------------------------------------------
diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml
index 30ba3d7..1effe55 100644
--- a/tagsync/src/main/resources/ranger-tagsync-site.xml
+++ b/tagsync/src/main/resources/ranger-tagsync-site.xml
@@ -70,22 +70,24 @@
 		<value />
 	</property>
 	<property>
-		<name>ranger.tagsync.keystore.filename</name>
-		<value>/etc/ranger/tagsync/conf/rangertagsync.jceks</value>
-	</property>
-	<property>
-		<name>ranger.tagsync.kerberos.keytab</name>
+		<name>ranger.tagsync.kerberos.principal</name>
 		<value />
 	</property>
 	<property>
-		<name>ranger.tagsync.kerberos.principal</name>
+		<name>ranger.tagsync.kerberos.keytab</name>
 		<value />
 	</property>
 	<property>
 		<name>ranger.tagsync.dest.ranger.username</name>
 		<value>rangertagsync</value>
 	</property>
-
-	
+	<property>
+		<name>ranger.tagsync.keystore.filename</name>
+		<value>/etc/ranger/tagsync/conf/rangertagsync.jceks</value>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.atlasrest.keystore.filename</name>
+		<value>/etc/ranger/tagsync/conf/atlasuser.jceks</value>
+	</property>
 
 </configuration>


Mime
View raw message