ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [24/50] [abbrv] incubator-ranger git commit: RANGER-660: implement TagSync process to populate Ranger with tag details
Date Wed, 28 Oct 2015 16:00:00 GMT
RANGER-660: implement TagSync process to populate Ranger with tag details

Signed-off-by: Madhan Neethiraj <madhan@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/eb874038
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/eb874038
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/eb874038

Branch: refs/heads/master
Commit: eb8740380fc53a353ddfc239455c5e9254476056
Parents: 9c42a8a
Author: Abhay Kulkarni <akulkarni@hortonworks.com>
Authored: Thu Sep 10 17:49:16 2015 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Mon Sep 28 09:57:15 2015 -0700

----------------------------------------------------------------------
 .../plugin/model/RangerTagResourceMap.java      |   4 +-
 .../apache/ranger/plugin/store/TagStore.java    |   2 +
 .../ranger/plugin/store/TagValidator.java       | 169 ++++--
 .../ranger/plugin/store/file/TagFileStore.java  |  26 +-
 .../apache/ranger/plugin/util/ServiceTags.java  |  18 +
 .../atlas-client-0.6-incubating-SNAPSHOT.jar    | Bin 0 -> 34445 bytes
 ...las-notification-0.6-incubating-SNAPSHOT.jar | Bin 0 -> 34596 bytes
 ...atlas-typesystem-0.6-incubating-SNAPSHOT.jar | Bin 0 -> 355378 bytes
 pom.xml                                         |   7 +
 .../java/org/apache/ranger/biz/TagDBStore.java  |  16 +
 .../apache/ranger/db/XXTagResourceMapDao.java   |   9 +
 .../ranger/rest/ServiceTagsProcessor.java       | 370 ++++++++++++
 .../java/org/apache/ranger/rest/TagREST.java    | 117 +++-
 .../apache/ranger/rest/TagRESTConstants.java    |   1 +
 .../service/RangerTagResourceMapService.java    |   6 +
 .../resources/META-INF/jpa_named_queries.xml    |   6 +
 src/main/assembly/tagsync.xml                   |  48 +-
 tagsync/conf.dist/log4j.xml                     |   4 +-
 .../conf.dist/ranger-tagsync-default-site.xml   |  36 --
 tagsync/conf.dist/ranger-tagsync-site.xml       |  67 +++
 tagsync/pom.xml                                 |   8 +-
 tagsync/samples/tags.json                       |  55 ++
 tagsync/scripts/ranger-tagsync-services.sh      |  22 +-
 .../java/org/apache/ranger/model/TagSink.java   |   3 -
 .../java/org/apache/ranger/model/TagSource.java |  13 +-
 .../apache/ranger/process/TagSyncConfig.java    | 132 ++---
 .../apache/ranger/process/TagSynchronizer.java  |  60 +-
 .../ranger/sink/policymgr/TagRESTSink.java      |  14 +-
 .../source/atlas/AtlasNotificationMapper.java   | 391 ++++++++++++
 .../ranger/source/atlas/TagAtlasSource.java     | 588 +++++++++++++++++++
 .../ranger/source/file/TagFileSource.java       | 183 +-----
 .../main/resources/ranger-tagsync-default.xml   |  91 +++
 .../src/main/resources/ranger-tagsync-site.xml  |  67 ---
 .../ranger/process/TestTagSynchronizer.java     |  17 +-
 34 files changed, 2024 insertions(+), 526 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java
index 3b69ee7..3ee0b3e 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java
@@ -71,9 +71,9 @@ public class RangerTagResourceMap extends RangerBaseModelObject {
 
 		sb.append("{ ");
 
-		sb.append("tagId=").append(tagId).append("} ");
+		sb.append("resourceId=").append(resourceId).append(", ");
 
-		sb.append("resourceId={").append(resourceId).append("} ");
+		sb.append("tagId=").append(tagId);
 
 		sb.append(" }");
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
index 1c48ea0..ed20c51 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
@@ -66,6 +66,8 @@ public interface TagStore {
 
     RangerTag getTagByGuid(String guid) throws Exception;
 
+    List<Long> getTagIdsForResourceId(Long resourceId) throws Exception;
+
     List<RangerTag> getTagsByType(String name) throws Exception;
 
     List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception;

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java
index 4eda166..c334e39 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java
@@ -35,10 +35,31 @@ public class TagValidator {
 		this.tagStore = tagStore;
 	}
 
-	public void preCreateTag(final RangerTag tag) throws Exception {
-		if (StringUtils.isBlank(tag.getType())) {
+	public RangerTagDef preCreateTagDef(final RangerTagDef tagDef, boolean updateIfExists) throws Exception {
+		String name = tagDef.getName();
+
+		if (StringUtils.isBlank(name)) {
+			throw new Exception("TagDef has no name");
+		}
+
+		RangerTagDef existing = tagStore.getTagDefByName(name);
+
+		return existing;
+	}
+
+	public RangerTag preCreateTag(final RangerTag tag) throws Exception {
+		if(StringUtils.isBlank(tag.getType()) ) {
 			throw new Exception("Tag has no type");
 		}
+
+		RangerTag ret = null;
+
+		String guid = tag.getGuid();
+		if (! StringUtils.isBlank(guid)) {
+			ret = tagStore.getTagByGuid(guid);
+		}
+
+		return ret;
 	}
 
 	public void preUpdateTag(final Long id, final RangerTag tag) throws Exception {
@@ -50,12 +71,14 @@ public class TagValidator {
 			throw new Exception("Invalid/null id");
 		}
 
-		RangerTag exist = tagStore.getTag(id);
+		RangerTag existing = tagStore.getTag(id);
 
-		if (exist == null) {
+		if (existing == null) {
 			throw new Exception("Attempt to update nonexistant tag, id=" + id);
 		}
-		tag.setId(exist.getId());
+
+		tag.setId(existing.getId());
+		tag.setGuid(existing.getGuid());
 	}
 
 	public void preUpdateTagByGuid(String guid, final RangerTag tag) throws Exception {
@@ -72,39 +95,27 @@ public class TagValidator {
 		tag.setGuid(existing.getGuid());
 	}
 
-	public void preUpdateTagByName(String name, final RangerTag tag) throws Exception {
-		if (StringUtils.isNotBlank(tag.getType())) {
-			throw new Exception("tag has no type");
-		}
-
-		List<RangerTag> exist = tagStore.getTagsByType(name);
-		if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) {
-			throw new Exception("Attempt to update nonexistent or multiple tags, name=" + name);
+	public RangerTag preDeleteTag(Long id) throws Exception {
+		if (id == null) {
+			throw new Exception("Invalid/null id");
 		}
 
-		RangerTag onlyTag = exist.get(0);
+		RangerTag existing = tagStore.getTag(id);
 
-		tag.setId(onlyTag.getId());
-		tag.setType(name);
-
-	}
-
-	public RangerTag preDeleteTag(Long id) throws Exception {
-		RangerTag exist;
-		exist = tagStore.getTag(id);
-		if (exist == null) {
+		if (existing == null) {
 			throw new Exception("Attempt to delete nonexistent tag, id=" + id);
 		}
 
-		List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(exist.getId());
+		List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(existing.getId());
 		if (CollectionUtils.isNotEmpty(associations)) {
 			throw new Exception("Attempt to delete tag which is associated with a service-resource, id=" + id);
 		}
-		return exist;
+		return existing;
 	}
 
 	public RangerTag preDeleteTagByGuid(String guid) throws Exception {
 		RangerTag exiting = tagStore.getTagByGuid(guid);
+
 		if (exiting == null) {
 			throw new Exception("Attempt to delete nonexistent tag, guid=" + guid);
 		}
@@ -116,37 +127,23 @@ public class TagValidator {
 		return exiting;
 	}
 
-	public RangerTag preDeleteTagByName(String name) throws Exception {
-		List<RangerTag> exist;
-		exist = tagStore.getTagsByType(name);
-		if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) {
-			throw new Exception("Attempt to delete nonexistent or multiple tags, name=" + name);
-		}
-		RangerTag ret = exist.get(0);
-		List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(ret.getId());
-		if (CollectionUtils.isNotEmpty(associations)) {
-			throw new Exception("Attempt to delete tag which is associated with a service-resource, name=" + name);
-		}
-		return ret;
-
-	}
+	public RangerServiceResource preCreateServiceResource(RangerServiceResource resource) throws Exception {
+		RangerServiceResource ret = null;
 
-	public void preCreateServiceResource(RangerServiceResource resource) throws Exception {
 		if (StringUtils.isBlank(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) {
 			throw new Exception("No serviceName or resource in RangerServiceResource");
 		}
 
-		RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
-
-		String resourceSignature = serializer.getSignature();
+		String guid = resource.getGuid();
+		if (! StringUtils.isBlank(guid)) {
+			ret = tagStore.getServiceResourceByGuid(guid);
+		}
 
-		RangerServiceResource exist = tagStore.getServiceResourceByResourceSignature(resourceSignature);
+		RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
 
-		if (exist != null) {
-			throw new Exception("Attempt to create existing resource, serviceName=" + resource.getServiceName());
-		}
+		resource.setResourceSignature(serializer.getSignature());
 
-		resource.setResourceSignature(resourceSignature);
+		return ret;
 	}
 
 	public void preUpdateServiceResource(Long id, RangerServiceResource resource) throws Exception {
@@ -158,15 +155,16 @@ public class TagValidator {
 			throw new Exception("Invalid/null id");
 		}
 
-		RangerServiceResource exist = tagStore.getServiceResource(id);
-		if (exist == null) {
+		RangerServiceResource existing = tagStore.getServiceResource(id);
+		if (existing == null) {
 			throw new Exception("Attempt to update nonexistent resource, id=" + id);
 		}
-		resource.setId(exist.getId());
 
 		RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
-		resource.setResourceSignature(serializer.getSignature());
 
+		resource.setId(existing.getId());
+		resource.setGuid(existing.getGuid());
+		resource.setResourceSignature(serializer.getSignature());
 	}
 
 	public void preUpdateServiceResourceByGuid(String guid, RangerServiceResource resource) throws Exception {
@@ -179,35 +177,40 @@ public class TagValidator {
 			throw new Exception("Attempt to update nonexistent resource, guid=" + guid);
 		}
 
+		RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
+
 		resource.setId(existing.getId());
 		resource.setGuid(guid);
-
-		RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
 		resource.setResourceSignature(serializer.getSignature());
 	}
 
 	public RangerServiceResource preDeleteServiceResource(Long id) throws Exception {
-		RangerServiceResource exist;
-		exist = tagStore.getServiceResource(id);
-		if (exist == null) {
+		RangerServiceResource existing = tagStore.getServiceResource(id);
+
+		if (existing == null) {
 			throw new Exception("Attempt to delete nonexistent resource, id=" + id);
 		}
-		List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(exist.getId());
+
+		List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(existing.getId());
 		if (CollectionUtils.isNotEmpty(associations)) {
 			throw new Exception("Attempt to delete serviceResource which is associated with a tag, id=" + id);
 		}
-		return exist;
+
+		return existing;
 	}
 
-	public RangerServiceResource preDeleteServiceResourceByGuid(String guid) throws Exception {
+	public RangerServiceResource preDeleteServiceResourceByGuid(String guid, boolean deleteReferences) throws Exception {
 		RangerServiceResource existing = tagStore.getServiceResourceByGuid(guid);
+
 		if (existing == null) {
 			throw new Exception("Attempt to delete nonexistent resource, guid=" + guid);
 		}
+
 		List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(existing.getId());
-		if (CollectionUtils.isNotEmpty(associations)) {
+		if (CollectionUtils.isNotEmpty(associations) && !deleteReferences) {
 			throw new Exception("Attempt to delete serviceResource which is associated with a tag, guid=" + guid);
 		}
+
 		return existing;
 	}
 
@@ -216,8 +219,9 @@ public class TagValidator {
 			throw new Exception("Both resourceGuid and resourceId need to be non-empty");
 		}
 
-		RangerTagResourceMap exist = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid);
-		if (exist != null) {
+		RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid);
+
+		if (existing != null) {
 			throw new Exception("Attempt to create existing association between resourceId=" + resourceGuid + " and tagId=" + tagGuid);
 		}
 
@@ -240,8 +244,35 @@ public class TagValidator {
 		return newTagResourceMap;
 	}
 
+	public RangerTagResourceMap preCreateTagResourceMapByIds(Long tagId, Long resourceId) throws Exception {
+		RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceId);
+
+		if (existing != null) {
+			throw new Exception("Attempt to create existing association between resourceId=" + resourceId + " and tagId=" + tagId);
+		}
+
+		RangerServiceResource existingServiceResource = tagStore.getServiceResource(resourceId);
+
+		if(existingServiceResource == null) {
+			throw new Exception("No resource found for id=" + resourceId);
+		}
+
+		RangerTag existingTag = tagStore.getTag(tagId);
+
+		if(existingTag == null) {
+			throw new Exception("No tag found for id=" + tagId);
+		}
+
+		RangerTagResourceMap newTagResourceMap = new RangerTagResourceMap();
+		newTagResourceMap.setResourceId(resourceId);
+		newTagResourceMap.setTagId(tagId);
+
+		return newTagResourceMap;
+	}
+
 	public RangerTagResourceMap preDeleteTagResourceMap(Long id) throws Exception {
 		RangerTagResourceMap existing = tagStore.getTagResourceMap(id);
+
 		if (existing == null) {
 			throw new Exception("Attempt to delete nonexistent tagResourceMap(id=" + id + ")");
 		}
@@ -251,6 +282,7 @@ public class TagValidator {
 
 	public RangerTagResourceMap preDeleteTagResourceMapByGuid(String guid) throws Exception {
 		RangerTagResourceMap existing = tagStore.getTagResourceMapByGuid(guid);
+
 		if (existing == null) {
 			throw new Exception("Attempt to delete nonexistent tagResourceMap(guid=" + guid + ")");
 		}
@@ -260,10 +292,21 @@ public class TagValidator {
 
 	public RangerTagResourceMap preDeleteTagResourceMap(String tagGuid, String resourceGuid) throws Exception {
 		RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid);
+
 		if (existing == null) {
 			throw new Exception("Attempt to delete nonexistent association between resourceId=" + resourceGuid + " and tagId=" + tagGuid);
 		}
 
 		return existing;
 	}
+
+	public RangerTagResourceMap preDeleteTagResourceMapByIds(Long tagId, Long resourceId) throws Exception {
+		RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceId);
+
+		if (existing == null) {
+			throw new Exception("Attempt to delete nonexistent association between resourceId=" + resourceId + " and tagId=" + tagId);
+		}
+
+		return existing;
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
index 27b4779..8f94164 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
@@ -31,6 +31,7 @@ import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
 import org.apache.ranger.plugin.model.*;
 import org.apache.ranger.plugin.store.AbstractTagStore;
 import org.apache.ranger.plugin.store.PList;
+import org.apache.ranger.plugin.store.RangerServiceResourceSignature;
 import org.apache.ranger.plugin.store.TagPredicateUtil;
 import org.apache.ranger.plugin.store.TagStore;
 import org.apache.ranger.plugin.util.SearchFilter;
@@ -42,6 +43,7 @@ public class TagFileStore extends AbstractTagStore {
 
 	public static final String PROPERTY_TAG_FILE_STORE_DIR = "ranger.tag.store.file.dir";
 
+
 	protected static final String FILE_PREFIX_TAG_DEF          = "ranger-tagdef-";
 	protected static final String FILE_PREFIX_TAG              = "ranger-tag-";
 	protected static final String FILE_PREFIX_RESOURCE         = "ranger-serviceresource-";
@@ -116,12 +118,6 @@ public class TagFileStore extends AbstractTagStore {
 			LOG.debug("==> TagFileStore.createTagDef(" + tagDef + ")");
 		}
 
-		RangerTagDef existing = getTagDefByName(tagDef.getName());
-
-		if (existing != null) {
-			throw new Exception(tagDef.getName() + ": tag-def already exists (id=" + existing.getId() + ")");
-		}
-
 		RangerTagDef ret = null;
 
 		try {
@@ -160,7 +156,7 @@ public class TagFileStore extends AbstractTagStore {
 				throw new Exception("tag-def does not exist: name=" + tagDef.getName());
 			}
 		} else {
-			existing = this.getTagDef(tagDef.getId());
+			existing = getTagDef(tagDef.getId());
 
 			if (existing == null) {
 				throw new Exception("tag-def does not exist: id=" + tagDef.getId());
@@ -504,6 +500,21 @@ public class TagFileStore extends AbstractTagStore {
 	}
 
 	@Override
+	public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception {
+		List<Long> ret = new ArrayList<Long>();
+
+		List<RangerTag> tags = getTagsForResourceId(resourceId);
+
+		if(CollectionUtils.isNotEmpty(tags)) {
+			for(RangerTag tag : tags) {
+				ret.add(tag.getId());
+			}
+		}
+
+		return ret;
+	}
+
+	@Override
 	public List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception {
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("==> TagFileStore.getTagsForResourceId(" + resourceId + ")");
@@ -1063,6 +1074,7 @@ public class TagFileStore extends AbstractTagStore {
 		}
 
 		ServiceTags ret = new ServiceTags();
+		ret.setOp(ServiceTags.OP_ADD_OR_UPDATE);
 
 		boolean tagsChanged = true;
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
index e56e942..146c3f9 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
@@ -44,6 +44,10 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
 public class ServiceTags implements java.io.Serializable {
 	private static final long serialVersionUID = 1L;
 
+	public static final String OP_ADD_OR_UPDATE = "add_or_update";
+	public static final String OP_DELETE        = "delete";
+
+	private String                      op;
 	private String                      serviceName;
 	private Long                        tagVersion;
 	private Date                        tagUpdateTime;
@@ -53,6 +57,13 @@ public class ServiceTags implements java.io.Serializable {
 	private Map<Long, List<Long>>       resourceToTagIds;
 
 	/**
+	 * @return the op
+	 */
+	public String getOp() {
+		return op;
+	}
+
+	/**
 	 * @return the serviceName
 	 */
 	public String getServiceName() {
@@ -60,6 +71,13 @@ public class ServiceTags implements java.io.Serializable {
 	}
 
 	/**
+	 * @param op the op to set
+	 */
+	public void setOp(String op) {
+		this.op = op;
+	}
+
+	/**
 	 * @param serviceName the serviceName to set
 	 */
 	public void setServiceName(String serviceName) {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar
new file mode 100644
index 0000000..feaaa38
Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar differ

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar
new file mode 100644
index 0000000..3c2ecb6
Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar differ

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar
new file mode 100644
index 0000000..8e5d1bf
Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar differ

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index aa40882..002885a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -255,6 +255,13 @@
                 <enabled>false</enabled>
             </snapshots>
           </repository>
+    <repository>
+      <id>repo</id>
+      <url>file://${user.home}/local-repo</url>
+      <snapshots>
+         <enabled>true</enabled>
+      </snapshots>
+  </repository>
   </repositories>
 	<dependencyManagement>
 		<dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 962bfde..98a6e87 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -719,6 +719,21 @@ public class TagDBStore extends AbstractTagStore {
 	}
 
 	@Override
+	public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> TagDBStore.getTagIdsForResourceId(" + resourceId + ")");
+		}
+
+		List<Long> ret = rangerTagResourceMapService.getTagIdsForResourceId(resourceId);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== TagDBStore.getTagIdsForResourceId(" + resourceId + "): count=" + (ret == null ? 0 : ret.size()));
+		}
+
+		return ret;
+	}
+
+	@Override
 	public List<RangerTagResourceMap> getTagResourceMapsForResourceId(Long resourceId) throws Exception {
 		if (LOG.isDebugEnabled()) {
 			LOG.debug("==> TagDBStore.getTagResourceMapsForResourceId(" + resourceId + ")");
@@ -875,6 +890,7 @@ public class TagDBStore extends AbstractTagStore {
 			}
 
 			ret = new ServiceTags();
+			ret.setOp(ServiceTags.OP_ADD_OR_UPDATE);
 			ret.setServiceName(xxService.getName());
 			ret.setTagVersion(xxService.getTagVersion());
 			ret.setTagUpdateTime(xxService.getTagUpdateTime());

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java
index 0cac1ef..72e0cd9 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java
@@ -70,6 +70,15 @@ public class XXTagResourceMapDao extends BaseDao<XXTagResourceMap> {
 		}
 	}
 
+	public List<Long> findTagIdsForResourceId(Long resourceId) {
+		try {
+			return getEntityManager().createNamedQuery("XXTagResourceMap.getTagIdsForResourceId", Long.class)
+					.setParameter("resourceId", resourceId).getResultList();
+		} catch (NoResultException e) {
+			return new ArrayList<Long>();
+		}
+	}
+
 	public List<XXTagResourceMap> findByTagId(Long tagId) {
 		if (tagId == null) {
 			return new ArrayList<XXTagResourceMap>();

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
new file mode 100644
index 0000000..7f0a92c
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.rest;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.plugin.model.RangerServiceResource;
+import org.apache.ranger.plugin.model.RangerTag;
+import org.apache.ranger.plugin.model.RangerTagDef;
+import org.apache.ranger.plugin.model.RangerTagResourceMap;
+import org.apache.ranger.plugin.store.RangerServiceResourceSignature;
+import org.apache.ranger.plugin.store.TagStore;
+import org.apache.ranger.plugin.util.ServiceTags;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ServiceTagsProcessor {
+	private static final Log LOG = LogFactory.getLog(ServiceTagsProcessor.class);
+
+	private final TagStore tagStore;
+
+	public ServiceTagsProcessor(TagStore tagStore) {
+		this.tagStore = tagStore;
+	}
+
+	public void process(ServiceTags serviceTags) throws Exception {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> ServiceTagsProcessor.process()");
+		}
+
+		if (tagStore != null && serviceTags != null) {
+			String op = serviceTags.getOp();
+
+			if (StringUtils.equalsIgnoreCase(op, ServiceTags.OP_ADD_OR_UPDATE)) {
+				addOrUpdate(serviceTags);
+			} else if (StringUtils.equalsIgnoreCase(op, ServiceTags.OP_DELETE)) {
+				delete(serviceTags);
+			} else {
+				LOG.error("Unknown op, op=" + op);
+			}
+		} else {
+			if(tagStore == null) {
+				LOG.error("tagStore is null!!");
+			}
+
+			if (serviceTags == null) {
+				LOG.error("No ServiceTags to import!!");
+			}
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== ServiceTagsProcessor.process()");
+		}
+	}
+
+	// Map tagdef, tag, serviceResource ids to created ids and use them in tag-resource-mapping
+	private void addOrUpdate(ServiceTags serviceTags) throws Exception {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> ServiceTagsProcessor.createOrUpdate()");
+		}
+
+		Map<Long, RangerTagDef>          tagDefsInStore   = new HashMap<Long, RangerTagDef>();
+		Map<Long, RangerTag>             tagsInStore      = new HashMap<Long, RangerTag>();
+		Map<Long, RangerServiceResource> resourcesInStore = new HashMap<Long, RangerServiceResource>();
+
+		boolean createOrUpdate = true;
+
+		if (MapUtils.isNotEmpty(serviceTags.getTagDefinitions())) {
+			RangerTagDef tagDef = null;
+
+			try {
+				for (Map.Entry<Long, RangerTagDef> entry : serviceTags.getTagDefinitions().entrySet()) {
+					tagDef = entry.getValue();
+
+					RangerTagDef existing = null;
+
+					if(createOrUpdate) {
+						if(StringUtils.isNotEmpty(tagDef.getGuid())) {
+							existing = tagStore.getTagDefByGuid(tagDef.getGuid());
+						}
+
+						if(existing == null && StringUtils.isNotEmpty(tagDef.getName())) {
+							existing = tagStore.getTagDefByName(tagDef.getName());
+						}
+					}
+
+					RangerTagDef tagDefInStore = null;
+
+					if(existing == null) {
+						tagDefInStore = tagStore.createTagDef(tagDef);
+					} else {
+						tagDef.setId(existing.getId());
+						tagDef.setGuid(existing.getGuid());
+
+						tagDefInStore = tagStore.updateTagDef(tagDef);
+					}
+
+					if(tagDefsInStore != null) {
+						tagDefsInStore.put(entry.getKey(), tagDefInStore);
+					}
+				}
+			} catch (Exception exception) {
+				LOG.error("createTagDef failed, tagDef=" + tagDef, exception);
+				throw exception;
+			}
+		}
+
+		if (MapUtils.isNotEmpty(serviceTags.getTags())) {
+			RangerTag tag = null;
+
+			try {
+				for (Map.Entry<Long, RangerTag> entry : serviceTags.getTags().entrySet()) {
+					tag = entry.getValue();
+
+					RangerTag existing = null;
+
+					if(createOrUpdate) {
+						if(StringUtils.isNotEmpty(tag.getGuid())) {
+							existing = tagStore.getTagByGuid(tag.getGuid());
+						}
+					}
+
+					RangerTag tagInStore = null;
+
+					if(existing == null) {
+						tagInStore = tagStore.createTag(tag);
+					} else {
+						tag.setId(existing.getId());
+						tag.setGuid(existing.getGuid());
+
+						tagInStore = tagStore.updateTag(tag);
+					}
+
+					if(tagsInStore != null) {
+						tagsInStore.put(entry.getKey(), tagInStore);
+					}
+				}
+			} catch (Exception exception) {
+				LOG.error("createTag failed, tag=" + tag, exception);
+				throw exception;
+			}
+		}
+
+		List<RangerServiceResource> resources = serviceTags.getServiceResources();
+		if (CollectionUtils.isNotEmpty(resources)) {
+			RangerServiceResource resource = null;
+
+			try {
+				for (int i = 0; i < resources.size(); i++) {
+					resource = resources.get(i);
+
+					RangerServiceResource existing          = null;
+					String                resourceSignature = null;
+					Long                  resourceId        = resource.getId();
+
+					if(createOrUpdate) {
+						if(StringUtils.isNotEmpty(resource.getGuid())) {
+							existing = tagStore.getServiceResourceByGuid(resource.getGuid());
+						}
+
+						if(existing == null) {
+							if(MapUtils.isNotEmpty(resource.getResourceElements())) {
+								RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
+
+								resourceSignature = serializer.getSignature();
+
+								existing = tagStore.getServiceResourceByResourceSignature(resourceSignature);
+							}
+						}
+
+						if(existing != null) {
+							resourceSignature = existing.getResourceSignature();
+						}
+					}
+
+					if(StringUtils.isEmpty(resourceSignature)) {
+						RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
+
+						resourceSignature = serializer.getSignature();
+					}
+
+					RangerServiceResource resourceInStore = null;
+
+					if (existing == null) {
+						resource.setResourceSignature(resourceSignature);
+
+						resourceInStore = tagStore.createServiceResource(resource);
+					} else {
+						resource.setId(existing.getId());
+						resource.setGuid(existing.getGuid());
+						resource.setResourceSignature(resourceSignature);
+
+						resourceInStore = tagStore.updateServiceResource(resource);
+					}
+
+					resourcesInStore.put(resourceId, resourceInStore);
+				}
+			} catch (Exception exception) {
+				LOG.error("createServiceResource failed, resource=" + resource, exception);
+				throw exception;
+			}
+		}
+
+		if (MapUtils.isNotEmpty(serviceTags.getResourceToTagIds())) {
+			for (Map.Entry<Long, List<Long>> entry : serviceTags.getResourceToTagIds().entrySet()) {
+				Long resourceId = entry.getKey();
+
+				RangerServiceResource resourceInStore = resourcesInStore.get(resourceId);
+
+				if (resourceInStore == null) {
+					LOG.error("Resource (id=" + resourceId + ") not found. Skipping tags update");
+					continue;
+				}
+
+				// Get all tags associated with this resourceId
+				List<Long> tagMapsToDelete = null;
+				try {
+					tagMapsToDelete = tagStore.getTagIdsForResourceId(resourceInStore.getId());
+				} catch (Exception exception) {
+					LOG.error("RangerTags cannot be retrieved for resource with guid=" + resourceInStore.getGuid());
+					throw exception;
+				}
+
+				List<Long> tagIds = entry.getValue();
+				try {
+					for (Long tagId : tagIds) {
+						RangerTag tagInStore = tagsInStore.get(tagId);
+
+						if (tagInStore == null) {
+							LOG.error("Tag (id=" + tagId + ") not found. Skipping addition of this tag for resource (id=" + resourceId + ")");
+							continue;
+						}
+
+						RangerTagResourceMap existing = null;
+
+						if(createOrUpdate) {
+							existing = tagStore.getTagResourceMapForTagAndResourceId(tagInStore.getId(), resourceInStore.getId());
+						}
+
+						if(existing == null) {
+							RangerTagResourceMap tagResourceMap = new RangerTagResourceMap();
+
+							tagResourceMap.setTagId(tagInStore.getId());
+							tagResourceMap.setResourceId(resourceInStore.getId());
+
+							tagResourceMap = tagStore.createTagResourceMap(tagResourceMap);
+						}
+
+						if(tagMapsToDelete != null) {
+							tagMapsToDelete.remove((Long)tagInStore.getId());
+						}
+					}
+				} catch (Exception exception) {
+					LOG.error("createRangerTagResourceMap failed", exception);
+					throw exception;
+				}
+
+				if (CollectionUtils.isNotEmpty(tagMapsToDelete)) {
+					Long tagId = null;
+
+					try {
+						for(int i = 0; i < tagMapsToDelete.size(); i++) {
+							tagId = tagMapsToDelete.get(i);
+
+							RangerTagResourceMap tagResourceMap = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceInStore.getId());
+
+							if(tagResourceMap != null) {
+								tagStore.deleteTagResourceMap(tagResourceMap.getId());
+							}
+
+							if (LOG.isDebugEnabled()) {
+								LOG.debug("Deleted tagResourceMap(tagId=" + tagId + ", resourceId=" + resourceId);
+							}
+						}
+					} catch(Exception exception) {
+						LOG.error("deleteTagResourceMap failed, tagId=" + tagId + ", resourceId=" + resourceId);
+						throw exception;
+					}
+				}
+			}
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== ServiceTagsProcessor.createOrUpdate()");
+		}
+	}
+
+	private void delete(ServiceTags serviceTags) throws Exception {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> ServiceTagsProcessor.delete()");
+		}
+
+		List<RangerServiceResource> serviceResources = serviceTags.getServiceResources();
+		if (CollectionUtils.isNotEmpty(serviceResources)) {
+			for (RangerServiceResource serviceResource : serviceResources) {
+				try {
+					RangerServiceResource objToDelete = tagStore.getServiceResourceByGuid(serviceResource.getGuid());
+
+					if(objToDelete != null) {
+						tagStore.deleteServiceResource(objToDelete.getId());
+					}
+				} catch (Exception exception) {
+					LOG.error("deleteServiceResourceByGuid failed, guid=" + serviceResource.getGuid(), exception);
+					throw exception;
+				}
+			}
+		}
+
+		Map<Long, RangerTag> tagsMap = serviceTags.getTags();
+		if (MapUtils.isNotEmpty(tagsMap)) {
+			for (Map.Entry<Long, RangerTag> entry : tagsMap.entrySet()) {
+				RangerTag tag = entry.getValue();
+				try {
+					RangerTag objToDelete = tagStore.getTagByGuid(tag.getGuid());
+
+					if(objToDelete != null) {
+						tagStore.deleteTag(objToDelete.getId());
+					}
+				} catch (Exception exception) {
+					LOG.error("deleteTag failed, guid=" + tag.getGuid(), exception);
+					throw exception;
+				}
+			}
+		}
+
+		Map<Long, RangerTagDef> tagDefsMap = serviceTags.getTagDefinitions();
+		if (MapUtils.isNotEmpty(tagDefsMap)) {
+			for (Map.Entry<Long, RangerTagDef> entry : tagDefsMap.entrySet()) {
+				RangerTagDef tagDef = entry.getValue();
+				try {
+					RangerTagDef objToDelete = tagStore.getTagDefByGuid(tagDef.getGuid());
+
+					if(objToDelete != null) {
+						tagStore.deleteTagDef(objToDelete.getId());
+					}
+				} catch (Exception exception) {
+					LOG.error("deleteTagDef failed, guid=" + tagDef.getGuid(), exception);
+					throw exception;
+				}
+			}
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== ServiceTagsProcessor.delete()");
+		}
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
index dfa4a64..378ff0b 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
@@ -29,6 +29,7 @@ import org.apache.ranger.plugin.model.RangerServiceResource;
 import org.apache.ranger.plugin.model.RangerTag;
 import org.apache.ranger.plugin.model.RangerTagResourceMap;
 import org.apache.ranger.plugin.model.RangerTagDef;
+import org.apache.ranger.plugin.store.TagStore;
 import org.apache.ranger.plugin.store.TagValidator;
 import org.apache.ranger.plugin.util.SearchFilter;
 import org.apache.ranger.plugin.util.ServiceTags;
@@ -75,19 +76,30 @@ public class TagREST {
         validator.setTagStore(tagStore);
 	}
 
+    TagStore getTagStore() {
+        return tagStore;
+    }
+
     @POST
     @Path(TagRESTConstants.TAGDEFS_RESOURCE)
     @Produces({ "application/json", "application/xml" })
     @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
-    public RangerTagDef createTagDef(RangerTagDef tagDef) {
+    public RangerTagDef createTagDef(RangerTagDef tagDef, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) {
         if(LOG.isDebugEnabled()) {
-            LOG.debug("==> TagREST.createTagDef(" + tagDef + ")");
+            LOG.debug("==> TagREST.createTagDef(" + tagDef + ", " + updateIfExists + ")");
         }
 
         RangerTagDef ret;
 
         try {
-            ret = tagStore.createTagDef(tagDef);
+            RangerTagDef exist = validator.preCreateTagDef(tagDef, updateIfExists);
+            if (exist == null) {
+                ret = tagStore.createTagDef(tagDef);
+            } else if (updateIfExists) {
+                ret = updateTagDef(exist.getId(), exist);
+            } else {
+                throw new Exception("tag-definition with Id " + exist.getId() + " already exists");
+            }
         } catch(Exception excp) {
             LOG.error("createTagDef(" + tagDef + ") failed", excp);
 
@@ -95,12 +107,13 @@ public class TagREST {
         }
 
         if(LOG.isDebugEnabled()) {
-            LOG.debug("<== TagREST.createTagDef(" + tagDef + "): " + ret);
+            LOG.debug("<== TagREST.createTagDef(" + tagDef + ", " + updateIfExists + "): " + ret);
         }
 
         return ret;
     }
 
+
     @PUT
     @Path(TagRESTConstants.TAGDEF_RESOURCE + "{id}")
     @Produces({ "application/json", "application/xml" })
@@ -327,16 +340,22 @@ public class TagREST {
     @Path(TagRESTConstants.TAGS_RESOURCE)
     @Produces({ "application/json", "application/xml" })
     @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
-    public RangerTag createTag(RangerTag tag) {
+    public RangerTag createTag(RangerTag tag, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) {
         if(LOG.isDebugEnabled()) {
-            LOG.debug("==> TagREST.createTag(" + tag + ")");
+            LOG.debug("==> TagREST.createTag(" + tag + ", " + updateIfExists + ")");
         }
 
         RangerTag ret;
 
         try {
-            validator.preCreateTag(tag);
-            ret = tagStore.createTag(tag);
+            RangerTag exist = validator.preCreateTag(tag);
+            if (exist == null) {
+                ret = tagStore.createTag(tag);
+            } else if (updateIfExists) {
+                ret = updateTag(exist.getId(), exist);
+            } else {
+                throw new Exception("tag with Id " + exist.getId() + " already exists");
+            }
         } catch(Exception excp) {
             LOG.error("createTag(" + tag + ") failed", excp);
 
@@ -344,7 +363,7 @@ public class TagREST {
         }
 
         if(LOG.isDebugEnabled()) {
-            LOG.debug("<== TagREST.createTag(" + tag + "): " + ret);
+            LOG.debug("<== TagREST.createTag(" + tag + ", " + updateIfExists + "): " + ret);
         }
 
         return ret;
@@ -553,16 +572,22 @@ public class TagREST {
     @Path(TagRESTConstants.RESOURCES_RESOURCE)
     @Produces({ "application/json", "application/xml" })
     @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
-    public RangerServiceResource createServiceResource(RangerServiceResource resource) {
+    public RangerServiceResource createServiceResource(RangerServiceResource resource, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) {
         if(LOG.isDebugEnabled()) {
-            LOG.debug("==> TagREST.createServiceResource(" + resource + ")");
+            LOG.debug("==> TagREST.createServiceResource(" + resource + ", " + updateIfExists + ")");
         }
 
         RangerServiceResource ret;
 
         try {
-            validator.preCreateServiceResource(resource);
-            ret = tagStore.createServiceResource(resource);
+            RangerServiceResource exist = validator.preCreateServiceResource(resource);
+            if (exist == null) {
+                ret = tagStore.createServiceResource(resource);
+            } else if (updateIfExists) {
+                ret = updateServiceResource(exist.getId(), exist);
+            } else {
+                throw new Exception("resource with Id " + exist.getId() + " already exists");
+            }
         } catch(Exception excp) {
             LOG.error("createServiceResource(" + resource + ") failed", excp);
 
@@ -570,7 +595,7 @@ public class TagREST {
         }
 
         if(LOG.isDebugEnabled()) {
-            LOG.debug("<== TagREST.createServiceResource(" + resource + "): " + ret);
+            LOG.debug("<== TagREST.createServiceResource(" + resource + ", " + updateIfExists + "): " + ret);
         }
 
         return ret;
@@ -650,22 +675,30 @@ public class TagREST {
     @Path(TagRESTConstants.RESOURCE_RESOURCE + "guid/{guid}")
     @Produces({ "application/json", "application/xml" })
     @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
-    public void deleteServiceResourceByGuid(@PathParam("guid") String guid) {
+    public void deleteServiceResourceByGuid(@PathParam("guid") String guid, @DefaultValue("false") @QueryParam("deleteReferences") boolean deleteReferences) {
         if(LOG.isDebugEnabled()) {
-            LOG.debug("==> TagREST.deleteServiceResourceByGuid(" + guid + ")");
+            LOG.debug("==> TagREST.deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ")");
         }
 
         try {
-            RangerServiceResource exist = validator.preDeleteServiceResourceByGuid(guid);
+            RangerServiceResource exist = validator.preDeleteServiceResourceByGuid(guid, deleteReferences);
+            if (deleteReferences) {
+                List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(exist.getGuid());
+                if (CollectionUtils.isNotEmpty(tagResourceMaps)) {
+                    for (RangerTagResourceMap tagResourceMap : tagResourceMaps) {
+                        deleteTagResourceMap(tagResourceMap.getId());
+                    }
+                }
+            }
             tagStore.deleteServiceResource(exist.getId());
         } catch(Exception excp) {
-            LOG.error("deleteServiceResourceByGuid(" + guid + ") failed", excp);
+            LOG.error("deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ") failed", excp);
 
             throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true);
         }
 
         if(LOG.isDebugEnabled()) {
-            LOG.debug("<== TagREST.deleteServiceResourceByGuid(" + guid + ")");
+            LOG.debug("<== TagREST.deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ")");
         }
     }
 
@@ -800,24 +833,31 @@ public class TagREST {
     @Path(TagRESTConstants.TAGRESOURCEMAPS_RESOURCE)
     @Produces({ "application/json", "application/xml" })
     @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
-    public RangerTagResourceMap createTagResourceMap(@QueryParam("tag-guid") String tagGuid, @QueryParam("resource-guid") String resourceGuid) {
+    public RangerTagResourceMap createTagResourceMap(@QueryParam("tag-guid") String tagGuid, @QueryParam("resource-guid") String resourceGuid,
+                                                     @DefaultValue("false") @QueryParam("lenient") boolean lenient) {
         if(LOG.isDebugEnabled()) {
-            LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " +  resourceGuid + ")");
+            LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " +  resourceGuid + ", " + lenient + ")");
         }
 
         RangerTagResourceMap tagResourceMap;
 
         try {
-            tagResourceMap = validator.preCreateTagResourceMap(tagGuid, resourceGuid);
-            tagResourceMap = tagStore.createTagResourceMap(tagResourceMap);
+            tagResourceMap = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid);
+            if (tagResourceMap == null) {
+                tagResourceMap = validator.preCreateTagResourceMap(tagGuid, resourceGuid);
+
+                tagResourceMap = tagStore.createTagResourceMap(tagResourceMap);
+            } else if (!lenient) {
+                throw new Exception("tagResourceMap with tag-guid=" + tagGuid + " and resource-guid=" + resourceGuid + " already exists");
+            }
         } catch(Exception excp) {
-            LOG.error("createTagResourceMap(" + tagGuid + ", " +  resourceGuid + ") failed", excp);
+            LOG.error("createTagResourceMap(" + tagGuid + ", " +  resourceGuid + ", " + lenient + ") failed", excp);
 
             throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true);
         }
 
         if(LOG.isDebugEnabled()) {
-            LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ")");
+            LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ", " + lenient + ")");
         }
 
         return tagResourceMap;
@@ -998,8 +1038,35 @@ public class TagREST {
         return ret;
     }
 
+    // This API is used by tag-sync to upload tag-objects
+
+    @PUT
+    @Path(TagRESTConstants.IMPORT_SERVICETAGS_RESOURCE)
+    @Produces({ "application/json", "application/xml" })
+    @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
+    public void importServiceTags(ServiceTags serviceTags) {
+        if(LOG.isDebugEnabled()) {
+            LOG.debug("==> TagREST.importServiceTags()");
+        }
+
+        try {
+
+            ServiceTagsProcessor serviceTagsProcessor = new ServiceTagsProcessor(tagStore);
+            serviceTagsProcessor.process(serviceTags);
+
+        } catch (Exception excp) {
+            LOG.error("importServiceTags() failed", excp);
+
+            throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true);
+
+        }
+        if(LOG.isDebugEnabled()) {
+            LOG.debug("<== TagREST.importServiceTags()");
+        }
+    }
 
     // This API is typically used by plug-in to get selected tagged resources from RangerAdmin
+
     @GET
     @Path(TagRESTConstants.TAGS_DOWNLOAD + "{serviceName}")
     @Produces({ "application/json", "application/xml" })

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
index 977e487..919f814 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
@@ -29,6 +29,7 @@ public class TagRESTConstants {
 	static final String RESOURCES_RESOURCE       = "/resources/";
 	static final String RESOURCE_RESOURCE        = "/resource/";
 	static final String TAGRESOURCEMAPS_RESOURCE = "/tagresourcemaps/";
+	static final String IMPORT_SERVICETAGS_RESOURCE = "/importservicetags/";
 	static final String TAGRESOURCEMAP_RESOURCE  = "/tagresourcemap/";
 	static final String TAGTYPES_RESOURCE        = "/types/";
 	static final String TAGTYPES_LOOKUP_RESOURCE = "/types/lookup/";

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
index d50a937..40d9d64 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
@@ -132,6 +132,12 @@ public class RangerTagResourceMapService extends RangerTagResourceMapServiceBase
 		return ret;
 	}
 
+	public List<Long> getTagIdsForResourceId(Long resourceId) {
+		List<Long> ret = daoMgr.getXXTagResourceMap().findTagIdsForResourceId(resourceId);
+
+		return ret;
+	}
+
 	public List<RangerTagResourceMap> getByResourceGuid(String resourceGuid) {
 		List<RangerTagResourceMap> ret = new ArrayList<RangerTagResourceMap>();
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
----------------------------------------------------------------------
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index a5f4a6c..993e7d6 100644
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -599,6 +599,12 @@
 				where obj.resourceId = res.id and res.serviceId = :serviceId
 				order by obj.resourceId</query>
 	</named-query>
+
+	<named-query name="XXTagResourceMap.getTagIdsForResourceId">
+		<query>select obj.tagId from XXTagResourceMap obj
+				where obj.resourceId = :resourceId
+		</query>
+	</named-query>
 	
 	<named-query name="XXTagDef.findByGuid">
 		<query>select obj from XXTagDef obj where obj.guid = :guid</query>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/src/main/assembly/tagsync.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml
index d7262ac..4f43851 100644
--- a/src/main/assembly/tagsync.xml
+++ b/src/main/assembly/tagsync.xml
@@ -47,14 +47,24 @@
 					<include>org.slf4j:slf4j-api</include>
 					<include>org.apache.hadoop:hadoop-common</include>
 					<include>org.apache.commons:commons-csv</include>
-					<include>org.apache.ranger:credentialbuilder</include>
-					<include>org.apache.ranger:ranger-util</include>
 					<include>commons-io:commons-io:jar:${commons.io.version}</include>
 					<include>org.apache.htrace:htrace-core</include>
 					<include>org.codehaus.jackson:jackson-core-asl</include>
 					<include>org.codehaus.jackson:jackson-jaxrs</include>
 					<include>org.codehaus.jackson:jackson-mapper-asl</include>
 					<include>org.codehaus.jackson:jackson-xc</include>
+					<include>org.apache.atlas:atlas-notification</include>
+					<include>org.apache.atlas:atlas-typesystem</include>
+					<include>org.apache.atlas:atlas-client</include>
+					<include>com.google.inject:guice</include>
+					<include>aopalliance:aopalliance</include>
+					<include>javax.inject:javax.inject</include>
+					<include>org.apache.kafka:kafka_2.10</include>
+					<include>org.apache.kafka:kafka-clients</include>
+					<include>com.yammer.metrics:metrics-core</include>
+					<include>org.scala-lang:scala-library</include>
+					<include>com.101tec:zkclient</include>
+					<include>org.apache.zookeeper:zookeeper</include>
 					<include>security_plugins.ranger-plugins-common:ranger-plugins-common</include>
 				</includes>
 				<unpack>false</unpack>
@@ -66,6 +76,29 @@
 	<include>org.apache.ranger:ranger-tagsync</include>
      </includes>
     </moduleSet>
+    <moduleSet>
+     <binaries>
+        <includeDependencies>false</includeDependencies>
+        <unpack>false</unpack>
+	    <directoryMode>755</directoryMode>
+	    <fileMode>644</fileMode>
+  		<dependencySets>
+   			<dependencySet>
+     			<outputDirectory>/lib</outputDirectory>
+				<includes>
+					<include>org.apache.ranger:credentialbuilder</include>
+					<include>org.apache.ranger:ranger-util</include>
+				</includes>
+				<unpack>false</unpack>
+  			</dependencySet>
+  		</dependencySets>
+        <outputDirectory>/lib</outputDirectory>
+     </binaries>
+     <includes>
+	<include>org.apache.ranger:credentialbuilder</include>
+	<include>org.apache.ranger:ranger-util</include>
+     </includes>
+    </moduleSet>
   </moduleSets> 	
   <fileSets>
 	<fileSet>
@@ -73,6 +106,15 @@
 	    	<fileMode>644</fileMode>
 		<outputDirectory>/conf.dist</outputDirectory>
 		<directory>tagsync/conf.dist</directory>
+		<excludes>
+			<exclude>etc</exclude>
+		</excludes>
+	</fileSet>
+	<fileSet>
+	    	<directoryMode>755</directoryMode>
+	    	<fileMode>644</fileMode>
+		<outputDirectory>/conf.dist/etc/ranger/data</outputDirectory>
+		<directory>tagsync/src/main/resources/etc/ranger/data</directory>
 	</fileSet>
 	<fileSet>
 	    	<directoryMode>755</directoryMode>
@@ -98,7 +140,7 @@
       <file>
 		<source>tagsync/scripts/initd</source>
 		<outputDirectory>/</outputDirectory>
-		<destName>ranger-usersync.sh</destName>
+		<destName>ranger-tagsync.sh</destName>
 		<fileMode>755</fileMode>
       </file>
    </files>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/log4j.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf.dist/log4j.xml b/tagsync/conf.dist/log4j.xml
index fb6986f..d455509 100644
--- a/tagsync/conf.dist/log4j.xml
+++ b/tagsync/conf.dist/log4j.xml
@@ -23,14 +23,14 @@
         <param name="file" value="${logdir}/tagsync.log" />
         <param name="DatePattern" value="'.'yyyy-MM-dd" />
         <layout class="org.apache.log4j.PatternLayout">
-		<param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"/>
+		<param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n"/>
         </layout>
    </appender>
 
   <appender name="console" class="org.apache.log4j.ConsoleAppender">
     <param name="Target" value="System.out"/>
     <layout class="org.apache.log4j.PatternLayout">
-	<param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"/>
+	<param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n"/>
     </layout>
   </appender>
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/ranger-tagsync-default-site.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf.dist/ranger-tagsync-default-site.xml b/tagsync/conf.dist/ranger-tagsync-default-site.xml
deleted file mode 100644
index b098740..0000000
--- a/tagsync/conf.dist/ranger-tagsync-default-site.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-	<property>
-		<name>ranger.tagsync.port</name>
-		<value>6161</value>
-	</property>
-	<property>
-		<name>ranger.tagsync.ssl</name>
-		<value>true</value>
-	</property>
-	<property>
-		<name>ranger.tagsync.enabled</name>
-		<value>true</value>
-	</property>
-	<property>
-		<name>ranger.tagsync.logdir</name>
-		<value>./log</value>
-	</property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/ranger-tagsync-site.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf.dist/ranger-tagsync-site.xml b/tagsync/conf.dist/ranger-tagsync-site.xml
new file mode 100644
index 0000000..21accd4
--- /dev/null
+++ b/tagsync/conf.dist/ranger-tagsync-site.xml
@@ -0,0 +1,67 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+
+<configuration>
+	<property>
+		<name>ranger.authentication.method</name>
+		<value>NONE</value>
+		<description></description>
+	</property>
+	<property>
+		<name>ranger.tagsync.tagadmin.rest.url</name>
+		<value>http://localhost:6080</value>
+		<description></description>
+	</property>
+	<property>
+		<name>ranger.tagsync.tagadmin.rest.ssl.config.file</name>
+		<value></value>
+		<description></description>
+	</property>
+	<property>
+		<name>ranger.tagsync.policymanager.basicauth.username</name>
+		<value>admin</value>
+		<description></description>
+	</property>
+	<property>
+		<name>ranger.tagsync.policymanager.basicauth.password</name>
+		<value>admin</value>
+		<description></description>
+	</property>
+	<property>
+		<name>ranger.tagsync.sleeptimeinmillisbetweensynccycle</name>
+		<value>60000</value>
+		<description></description>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.file</name>
+		<value>/etc/ranger/data/tags.json</value>
+		<description></description>
+	</property>
+	<property>
+		<name>ranger.tagsync.source.impl.class</name>
+		<value>org.apache.ranger.source.file.TagFileSource</value>
+		<description></description>
+	</property>
+	<property>
+		<name>ranger.tagsync.sink.impl.class</name>
+		<value>org.apache.ranger.sink.policymgr.TagRESTSink</value>
+		<description></description>
+	</property>
+	<property>
+		<name>atlas.endpoint</name>
+		<value>http://localhost:21000/</value>
+		<description></description>
+	</property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/pom.xml
----------------------------------------------------------------------
diff --git a/tagsync/pom.xml b/tagsync/pom.xml
index 83011d2..18c684c 100644
--- a/tagsync/pom.xml
+++ b/tagsync/pom.xml
@@ -86,7 +86,6 @@
             <artifactId>ranger-plugins-common</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <!--
         <dependency>
             <groupId>com.google.inject</groupId>
             <artifactId>guice</artifactId>
@@ -96,17 +95,22 @@
             <groupId>org.apache.atlas</groupId>
             <artifactId>atlas-notification</artifactId>
             <version>0.6-incubating-SNAPSHOT</version>
+            <scope>system</scope>
+            <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar</systemPath>
         </dependency>
         <dependency>
             <groupId>org.apache.atlas</groupId>
             <artifactId>atlas-typesystem</artifactId>
             <version>0.6-incubating-SNAPSHOT</version>
+            <scope>system</scope>
+            <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar</systemPath>
         </dependency>
         <dependency>
             <groupId>org.apache.atlas</groupId>
             <artifactId>atlas-client</artifactId>
             <version>0.6-incubating-SNAPSHOT</version>
+            <scope>system</scope>
+            <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar</systemPath>
         </dependency>
-        -->
     </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/samples/tags.json
----------------------------------------------------------------------
diff --git a/tagsync/samples/tags.json b/tagsync/samples/tags.json
new file mode 100644
index 0000000..1b24fc8
--- /dev/null
+++ b/tagsync/samples/tags.json
@@ -0,0 +1,55 @@
+{
+  "op": "add_or_update",
+  "serviceName": "cl1_hive",
+  "tagVersion": 1,
+  "tagUpdateTime": "20150924-22:26:33.000-+0000",
+  "tagDefinitions": {
+     "1":{"id":1, "guid":"tagdef-1", "name":"PII",        "attributeDefs":[]},
+     "2":{"id":2, "guid":"tagdef-2", "name":"EXPIRES_ON", "attributeDefs":[{"name":"expiry_date", "type":"date"}]},
+     "3":{"id":3, "guid":"tagdef-3", "name":"FINANCE",    "attributeDefs":[]},
+     "4":{"id":4, "guid":"tagdef-4", "name":"AUDIT",      "attributeDefs":[]}
+   },
+  "tags": {
+     "1":{"id":1, "guid":"tag-1", "type":"PII",        "attributes":{}},
+     "2":{"id":2, "guid":"tag-2", "type":"EXPIRES_ON", "attributes":{"expiry_date":"2015/08/31"}},
+     "3":{"id":3, "guid":"tag-3", "type":"FINANCE",    "attributes":{}},
+     "4":{"id":4, "guid":"tag-4", "type":"AUDIT",      "attributes":{}}
+   },
+   "serviceResources": [
+     {
+       "id":1,
+       "guid":"resource-1",
+       "serviceName":"cl1_hive",
+       "resourceElements":{
+         "database":{"values":["hr"]},
+         "table":{"values":["employee"]},
+         "column":{"values":["ssn"]}
+       }
+     },
+     {
+       "id":2,
+       "guid":"resource-2",
+       "serviceName":"cl1_hive",
+       "resourceElements":{
+         "database":{"values":["finance"]},
+         "table":{"values":["tax_2010"]}
+       }
+     },
+     {
+       "id":3,
+       "guid":"resource-3",
+       "serviceName":"cl1_hive",
+       "resourceElements":{
+         "database":{"values":["finance"]},
+         "table":{"values":["tax_2010"]},
+         "column":{"values":["ssn"]}
+       }
+     }
+   ],
+
+   "resourceToTagIds": {
+     "1":[1],
+     "2":[2],
+     "3":[3]
+   }
+}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/scripts/ranger-tagsync-services.sh
----------------------------------------------------------------------
diff --git a/tagsync/scripts/ranger-tagsync-services.sh b/tagsync/scripts/ranger-tagsync-services.sh
index 5ca5b95..2f4b255 100755
--- a/tagsync/scripts/ranger-tagsync-services.sh
+++ b/tagsync/scripts/ranger-tagsync-services.sh
@@ -27,8 +27,13 @@ realScriptDir=`dirname $realScriptPath`
 cd $realScriptDir
 cdir=`pwd`
 
-pidf=/var/run/ranger/tagsync.pid
+pidd=/var/run/ranger
 
+if [ -d $pidd ]; then
+	mkdir -p $pidd
+fi
+
+pidf=${pidd}/tagsync.pid
 
 if [ "${action}" == "START" ]; then
 
@@ -47,9 +52,13 @@ if [ "${action}" == "START" ]; then
         	export PATH=$JAVA_HOME/bin:$PATH
 	fi
 
-        logdir=/var/log/ranger/tagsync
+    logdir=/var/log/ranger/tagsync
+
+	if [ ! -d $logdir ]; then
+		mkdir -p $logdir
+	fi
 
-	cp="${cdir}/dist/*:${cdir}/lib/*:${cdir}/conf"
+	cp="${cdir}/conf.dist:${cdir}/dist/*:${cdir}/lib/*"
 
     if [ -f $pidf ]; then
             PID=`cat $pidf`
@@ -81,7 +90,6 @@ if [ "${action}" == "START" ]; then
 elif [ "${action}" == "STOP" ]; then
 
     if [ -f $pidf ]; then
-            pidf=/var/run/ranger/tagsync.pid
 	        PID=`cat $pidf` > /dev/null 2>&1
             kill -9 $PID > /dev/null 2>&1
             rm -f $pidf
@@ -103,8 +111,8 @@ elif [ "${action}" == "VERSION" ]; then
 	java -cp ranger-util-*.jar org.apache.ranger.common.RangerVersionInfo
 	exit
 else 
-	        echo "Invalid argument [$1];"
-        echo "Usage: Only start | stop | restart | version, are supported."
-        exit;
+	echo "Invalid argument [$1];"
+    echo "Usage: Only start | stop | restart | version, are supported."
+    exit;
 fi
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/model/TagSink.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java
index 74ea5cf..5256570 100644
--- a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java
+++ b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java
@@ -26,9 +26,6 @@ import java.util.Map;
 import java.util.Properties;
 
 
-/**
- * Created by akulkarni on 9/10/15.
- */
 public interface TagSink extends TagStore {
 	boolean initialize(Properties properties);
 	void uploadServiceTags(ServiceTags serviceTags) throws Exception;

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/model/TagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java
index 568d3c7..0ccceae 100644
--- a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java
@@ -26,9 +26,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-/**
- * Created by akulkarni on 9/10/15.
- */
 public interface TagSource {
 
 	boolean initialize(Properties properties);
@@ -37,16 +34,8 @@ public interface TagSource {
 
 	void updateSink() throws Exception;
 
-	void start();
+	Thread start();
 
 	boolean isChanged();
 
-	List<RangerTagDef> fetchAllTagDefs(String syncSentinel) throws Exception;
-
-	List<RangerTagDef> receiveUpdatesToTagDefs() throws Exception;
-
-	List<RangerTagResourceMap> fetchAllTaggedEntities() throws Exception;
-
-	List<RangerTagResourceMap> receiveUpdatesToTaggedEntities() throws Exception;
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java
index 1c995b1..c80eff9 100644
--- a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java
+++ b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java
@@ -20,27 +20,20 @@
 package org.apache.ranger.process;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.log4j.Logger;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import java.io.*;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Properties;
 
-/**
- * Created by akulkarni on 9/11/15.
- */
-public class TagSyncConfig {
+public class TagSyncConfig extends Configuration {
 	private static final Logger LOG = Logger.getLogger(TagSyncConfig.class) ;
 
 	public static final String CONFIG_FILE = "ranger-tagsync-site.xml";
 
-	public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default-site.xml";
+	public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml";
 
 	public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled" ;
 
@@ -50,9 +43,9 @@ public class TagSyncConfig {
 
 	public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir" ;
 
-	private static final String TAGSYNC_PM_URL_PROP = 	"ranger.tagsync.policymanager.baseURL";
+	private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.tagadmin.rest.url";
 
-	private static final String TAGSYNC_PM_SSL_CONFIG_FILE_PROP = "ranger.tagsync.policymanager.ssl.config.file";
+	private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.tagadmin.rest.ssl.config.file";
 
 	private static final String TAGSYNC_PM_SSL_BASICAUTH_USERNAME_PROP = "ranger.tagsync.policymanager.basicauth.username";
 
@@ -68,9 +61,13 @@ public class TagSyncConfig {
 
 	private static final String TAGSYNC_SOURCE_ATLAS_PROP = "atlas.endpoint";
 
-	private static volatile TagSyncConfig instance = null;
+	private static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas.";
+
+	private static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service";
 
-	private Properties prop = new Properties() ;
+	private static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_";
+
+	private static volatile TagSyncConfig instance = null;
 
 	public static TagSyncConfig getInstance() {
 	/*
@@ -85,12 +82,11 @@ public class TagSyncConfig {
 		}
 	*/
 		TagSyncConfig newConfig = new TagSyncConfig();
-		newConfig.init();
 		return newConfig;
 	}
 
 	public Properties getProperties() {
-		return prop;
+		return getProps();
 	}
 
 	public static InputStream getFileInputStream(String path) throws FileNotFoundException {
@@ -158,7 +154,7 @@ public class TagSyncConfig {
 						LOG.error(path + " is not a file", exception);
 					}
 				} else {
-					LOG.error("URL not found for " + path + " or no privilege for reading file " + path);
+					LOG.warn("URL not found for " + path + " or no privilege for reading file " + path);
 				}
 			}
 		}
@@ -170,16 +166,10 @@ public class TagSyncConfig {
 	public String toString() {
 		StringBuffer sb = new StringBuffer();
 
-		sb.append("CONFIG_FILE=").append(CONFIG_FILE).append(", ")
-				.append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append("\n");
-
-		ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-		PrintStream printStream = new PrintStream(outputStream);
-		prop.list(printStream);
-		printStream.close();
-		sb.append(outputStream.toString());
+		sb.append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append(", ")
+				.append("CONFIG_FILE=").append(CONFIG_FILE).append("\n\n");
 
-		return sb.toString();
+		return sb.toString() + super.toString();
 	}
 
 	static public boolean isTagSyncEnabled(Properties prop) {
@@ -217,13 +207,13 @@ public class TagSyncConfig {
 		return val;
 	}
 
-	static public String getPolicyMgrUrl(Properties prop) {
-		String val = prop.getProperty(TAGSYNC_PM_URL_PROP);
+	static public String getTagAdminRESTUrl(Properties prop) {
+		String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP);
 		return val;
 	}
 
-	static public String getPolicyMgrSslConfigFile(Properties prop) {
-		String val = prop.getProperty(TAGSYNC_PM_SSL_CONFIG_FILE_PROP);
+	static public String getTagAdminRESTSslConfigFile(Properties prop) {
+		String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP);
 		return val;
 	}
 
@@ -251,75 +241,39 @@ public class TagSyncConfig {
 		return "";
 	}
 
+	static public String getServiceName(String componentName, String instanceName, Properties prop) {
+		String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName
+				+ ".instance." + instanceName
+				+ TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX;
+		String val = prop.getProperty(propName);
+		if (StringUtils.isBlank(val)) {
+			val = instanceName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + componentName;
+		}
+		return val;
+	}
+
 	private TagSyncConfig() {
+		super(false);
 		init() ;
 	}
 
 	private void init() {
-		readConfigFile(CONFIG_FILE);
 		readConfigFile(DEFAULT_CONFIG_FILE);
+		readConfigFile(CONFIG_FILE);
 	}
 
 	private void readConfigFile(String fileName) {
-		try {
-			InputStream in = getFileInputStream(fileName);
-			if (in != null) {
-				try {
-					DocumentBuilderFactory xmlDocumentBuilderFactory = DocumentBuilderFactory
-							.newInstance();
-					xmlDocumentBuilderFactory.setIgnoringComments(true);
-					xmlDocumentBuilderFactory.setNamespaceAware(true);
-					DocumentBuilder xmlDocumentBuilder = xmlDocumentBuilderFactory
-							.newDocumentBuilder();
-					Document xmlDocument = xmlDocumentBuilder.parse(in);
-					xmlDocument.getDocumentElement().normalize();
-
-					NodeList nList = xmlDocument
-							.getElementsByTagName("property");
-
-					for (int temp = 0; temp < nList.getLength(); temp++) {
-
-						Node nNode = nList.item(temp);
-
-						if (nNode.getNodeType() == Node.ELEMENT_NODE) {
-
-							Element eElement = (Element) nNode;
-
-							String propertyName = "";
-							String propertyValue = "";
-							if (eElement.getElementsByTagName("name").item(
-									0) != null) {
-								propertyName = eElement
-										.getElementsByTagName("name")
-										.item(0).getTextContent().trim();
-							}
-							if (eElement.getElementsByTagName("value")
-									.item(0) != null) {
-								propertyValue = eElement
-										.getElementsByTagName("value")
-										.item(0).getTextContent().trim();
-							}
-
-							if (prop.get(propertyName) != null) {
-								prop.remove(propertyName) ;
-							}
-
-							prop.put(propertyName, propertyValue);
 
-						}
-					}
-				}
-				finally {
-					try {
-						in.close() ;
-					}
-					catch(IOException ioe) {
-						// Ignore IOE when closing stream
-					}
-				}
+		if (StringUtils.isNotBlank(fileName)) {
+			String fName = getResourceFileName(fileName);
+			if (StringUtils.isBlank(fName)) {
+				LOG.warn("Cannot find configuration file " + fileName + " in the classpath");
+			} else {
+				LOG.info("Loading configuration from " + fName);
+				addResource(fileName);
 			}
-		} catch (Throwable e) {
-			throw new RuntimeException("Unable to load configuration file [" + CONFIG_FILE + "]", e) ;
+		} else {
+			LOG.error("Configuration fileName is null");
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java
index c7e8d6e..8ab2a1a 100644
--- a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java
+++ b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java
@@ -24,16 +24,14 @@ import org.apache.log4j.Logger;
 import org.apache.ranger.model.TagSink;
 import org.apache.ranger.model.TagSource;
 
+import java.util.Map;
 import java.util.Properties;
 
-/**
- * Created by akulkarni on 9/11/15.
- */
 public class TagSynchronizer implements Runnable {
 
 	private static final Logger LOG = Logger.getLogger(TagSynchronizer.class);
 
-	private final static int MAX_INIT_RETRIES = 1;
+	private final static int MAX_INIT_RETRIES = 5;
 
 	private boolean shutdownFlag = false;
 	private TagSink tagSink = null;
@@ -46,12 +44,6 @@ public class TagSynchronizer implements Runnable {
 		TagSyncConfig config = TagSyncConfig.getInstance();
 		Properties props = config.getProperties();
 
-		LOG.info("--------------------------------");
-		LOG.info("");
-		LOG.info("Ranger-TagSync Configuration: {\n" + config + "\n}");
-		LOG.info("");
-		LOG.info("--------------------------------");
-
 		TagSynchronizer tagSynchronizer = new TagSynchronizer(props);
 
 		tagSynchronizer.run();
@@ -86,15 +78,28 @@ public class TagSynchronizer implements Runnable {
 
 			if (initDone) {
 
-				tagSource.start();
+				Thread tagSourceThread = tagSource.start();
 
-				while (!shutdownFlag) {
-					try {
-						LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds");
-						Thread.sleep(sleepTimeBetweenCycleInMillis);
-					} catch (InterruptedException e) {
-						LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e);
+				if (tagSourceThread != null) {
+					while (!shutdownFlag) {
+						try {
+							LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds");
+							Thread.sleep(sleepTimeBetweenCycleInMillis);
+						} catch (InterruptedException e) {
+							LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e);
+						}
 					}
+					if (shutdownFlag) {
+						LOG.info("Interrupting tagSourceThread...");
+						tagSourceThread.interrupt();
+						try {
+							tagSourceThread.join();
+						} catch (InterruptedException interruptedException) {
+							LOG.error("tagSourceThread.join() was interrupted");
+						}
+					}
+				} else {
+					LOG.error("Could not start tagSource monitoring thread");
 				}
 			} else {
 				LOG.error("Failed to initialize TagSynchonizer after " + MAX_INIT_RETRIES + " retries. Exiting thread");
@@ -121,6 +126,8 @@ public class TagSynchronizer implements Runnable {
 
 		for (int initRetries = 0; initRetries < MAX_INIT_RETRIES && !ret; initRetries++) {
 
+			printConfigurationProperties();
+
 			ret = init();
 
 			if (!ret) {
@@ -128,6 +135,7 @@ public class TagSynchronizer implements Runnable {
 				try {
 					LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds");
 					Thread.sleep(sleepTimeBetweenCycleInMillis);
+					properties = TagSyncConfig.getInstance().getProperties();
 				} catch (Exception e) {
 					LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to initialize tag source/sink", e);
 				}
@@ -181,7 +189,23 @@ public class TagSynchronizer implements Runnable {
 	}
 
 	public void shutdown(String reason) {
-		LOG.error("Received shutdown(), reason=" + reason);
+		LOG.info("Received shutdown(), reason=" + reason);
 		this.shutdownFlag = true;
 	}
+
+	public void printConfigurationProperties() {
+		LOG.info("--------------------------------");
+		LOG.info("");
+		LOG.info("Ranger-TagSync Configuration: {\n");
+		if (MapUtils.isNotEmpty(properties)) {
+			for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+				LOG.info("\tProperty-Name:" + entry.getKey());
+				LOG.info("\tProperty-Value:" + entry.getValue());
+				LOG.info("\n");
+			}
+		}
+		LOG.info("\n}");
+		LOG.info("");
+		LOG.info("--------------------------------");
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java
index 98410d6..e934ce4 100644
--- a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java
+++ b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java
@@ -39,9 +39,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
-/**
- * Created by akulkarni on 9/11/15.
- */
 public class TagRESTSink implements TagSink {
 	private static final Log LOG = LogFactory.getLog(TagRESTSink.class);
 
@@ -57,8 +54,6 @@ public class TagRESTSink implements TagSink {
 	private static final String REST_URL_TAG_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tag/" ;
 	private static final String REST_URL_TAGRESOURCEMAP_IDS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagresourcemapids/";
 	private static final String REST_URL_IMPORT_SERVICETAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/importservicetags/";
-	public static final String REST_URL_IMPORT_SERVICETAGS_PARAM = "op";
-
 
 	private RangerRESTClient tagRESTClient = null;
 
@@ -73,8 +68,8 @@ public class TagRESTSink implements TagSink {
 
 		boolean ret = false;
 
-		String restUrl       = TagSyncConfig.getPolicyMgrUrl(properties);
-		String sslConfigFile = TagSyncConfig.getPolicyMgrSslConfigFile(properties);
+		String restUrl       = TagSyncConfig.getTagAdminRESTUrl(properties);
+		String sslConfigFile = TagSyncConfig.getTagAdminRESTSslConfigFile(properties);
 		String userName = TagSyncConfig.getPolicyMgrUserName(properties);
 		String password = TagSyncConfig.getPolicyMgrPassword(properties);
 
@@ -267,6 +262,11 @@ public class TagRESTSink implements TagSink {
 	}
 
 	@Override
+	public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception {
+		throw new Exception("Not implemented");
+	}
+
+	@Override
 	public List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception {
 		throw new Exception("Not implemented");
 	}


Mime
View raw message