Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 60691200498 for ; Tue, 29 Aug 2017 18:15:02 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 5E2BC16701C; Tue, 29 Aug 2017 16:15:02 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 36EA7167017 for ; Tue, 29 Aug 2017 18:15:00 +0200 (CEST) Received: (qmail 48543 invoked by uid 500); 29 Aug 2017 16:14:59 -0000 Mailing-List: contact commits-help@sentry.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@sentry.apache.org Delivered-To: mailing list commits@sentry.apache.org Received: (qmail 48534 invoked by uid 99); 29 Aug 2017 16:14:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 29 Aug 2017 16:14:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F2A70E0AF6; Tue, 29 Aug 2017 16:14:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: spena@apache.org To: commits@sentry.apache.org Message-Id: <9afc78afe68b46ec9a374a973369c369@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: sentry git commit: SENTRY-1895: Sentry should handle the case of multiple notifications with the same ID (Sergio Pena, reviewed by Na Li, Alexander Kolbasov) Date: Tue, 29 Aug 2017 16:14:58 +0000 (UTC) archived-at: Tue, 29 Aug 2017 16:15:02 -0000 Repository: sentry Updated Branches: refs/heads/master 19a70338a -> ab906afa2 SENTRY-1895: Sentry should handle the case of multiple notifications with the same ID (Sergio Pena, reviewed by Na Li, Alexander Kolbasov) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/ab906afa Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/ab906afa Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/ab906afa Branch: refs/heads/master Commit: ab906afa2bba09fd007172d522ff808d7257ef32 Parents: 19a7033 Author: Sergio Pena Authored: Tue Aug 29 11:14:07 2017 -0500 Committer: Sergio Pena Committed: Tue Aug 29 11:14:07 2017 -0500 ---------------------------------------------------------------------- .../apache/sentry/hdfs/UniquePathsUpdate.java | 62 +++++++++++++++++++ .../apache/sentry/hdfs/TestDeltaRetriever.java | 4 +- .../hdfs/TestSentryHDFSServiceProcessor.java | 2 +- .../db/service/model/MSentryPathChange.java | 27 ++++++--- .../provider/db/service/model/package.jdo | 14 ++++- .../persistent/DeltaTransactionBlock.java | 6 +- .../db/service/persistent/SentryStore.java | 13 ++-- .../service/thrift/NotificationProcessor.java | 64 ++++++++++---------- .../main/resources/008-SENTRY-1569.derby.sql | 5 +- .../main/resources/008-SENTRY-1569.mysql.sql | 9 ++- .../main/resources/008-SENTRY-1569.oracle.sql | 7 ++- .../main/resources/008-SENTRY-1569.postgres.sql | 7 ++- .../src/main/resources/sentry-db2-2.0.0.sql | 7 ++- .../src/main/resources/sentry-derby-2.0.0.sql | 6 +- .../src/main/resources/sentry-mysql-2.0.0.sql | 7 ++- .../src/main/resources/sentry-oracle-2.0.0.sql | 7 ++- .../main/resources/sentry-postgres-2.0.0.sql | 7 ++- .../sentry-upgrade-db2-1.7.0-to-2.0.0.sql | 5 +- .../db/service/model/TestMSentryUtil.java | 12 ++-- .../db/service/persistent/TestSentryStore.java | 43 +++++++------ .../sentry/service/thrift/TestHMSFollower.java | 26 ++++---- .../thrift/TestNotificationProcessor.java | 30 ++++----- 22 files changed, 248 insertions(+), 122 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UniquePathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UniquePathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UniquePathsUpdate.java new file mode 100644 index 0000000..7dae2f5 --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/UniquePathsUpdate.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.hdfs; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.hive.metastore.api.NotificationEvent; + +/** + * A wrapper class over the PathsUpdate class that also stores a unique hash that identifies + * unique HMS notification events. + * + * HMS notification events may have duplicated events IDs. To keep this unique, a SHA-1 hash + * calculation is done and persisted on the SENTRY_PATH_CHANGE table. + * + * TODO: Stop using this class once HIVE-16886 is fixed. + */ +public class UniquePathsUpdate extends PathsUpdate { + private final String eventHash; + + public UniquePathsUpdate(NotificationEvent event, boolean hasFullImage) { + super(event.getEventId(), hasFullImage); + this.eventHash = sha1(event); + } + + public UniquePathsUpdate(String eventHash, long eventId, boolean hasFullImage) { + super(eventId, hasFullImage); + this.eventHash = eventHash; + } + + public String getEventHash() { + return eventHash; + } + + private String sha1(NotificationEvent event) { + StringBuilder sb = new StringBuilder(); + + sb.append(event.getEventId()); + sb.append(event.getEventTime()); + sb.append(event.getEventType()); + sb.append(event.getDbName()); + sb.append(event.getTableName()); + sb.append(event.getMessage()); + + return DigestUtils.shaHex(sb.toString()); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java index 7ea75a0..63b8caf 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestDeltaRetriever.java @@ -55,8 +55,8 @@ public class TestDeltaRetriever { List pathsUpdates; List deltaPathChanges = Arrays.asList( - new MSentryPathChange(1, new PathsUpdate(1, true)), - new MSentryPathChange(2, new PathsUpdate(2, false)) + new MSentryPathChange(1, "u1", new PathsUpdate(1, true)), + new MSentryPathChange(2, "u2", new PathsUpdate(2, false)) ); Mockito.when(sentryStoreMock.getMSentryPathChanges(Mockito.anyLong())) http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java index f2368b7..e403d7c 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java +++ b/sentry-hdfs/sentry-hdfs-service/src/test/java/org/apache/sentry/hdfs/TestSentryHDFSServiceProcessor.java @@ -106,7 +106,7 @@ public class TestSentryHDFSServiceProcessor { Mockito.when(sentryStoreMock.pathChangeExists(2)) .thenReturn(true); Mockito.when(sentryStoreMock.getMSentryPathChanges(2)) - .thenReturn(Arrays.asList(new MSentryPathChange(3, new PathsUpdate(3, 1, false)))); + .thenReturn(Arrays.asList(new MSentryPathChange(3, "u3", new PathsUpdate(3, 1, false)))); Mockito.when(sentryStoreMock.getLastProcessedPermChangeID()) .thenReturn(3L); http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java index 58878e5..bb5ff21 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/MSentryPathChange.java @@ -66,15 +66,24 @@ public class MSentryPathChange implements MSentryChange { // Path change in JSON format. private String pathChange; private long createTimeMs; - private long notificationID; + private String notificationHash; - public MSentryPathChange(long changeID, PathsUpdate pathChange) throws TException { + public MSentryPathChange(long changeID, String notificationHash, PathsUpdate pathChange) throws TException { // Each PathsUpdate maps to a MSentryPathChange object. // The PathsUpdate is generated from a HMS notification log, // the notification ID is stored as seqNum and // the notification update is serialized as JSON string. this.changeID = changeID; - this.notificationID = pathChange.getSeqNum(); + + /* + * notificationHash is a unique identifier for the HMS notification used to prevent + * the same HMS notification message to be processed twice. + * The current HMS code may send different notifications messages with the same ID. To + * keep this ID unique, we calculate the SHA-1 hash of the full message received. + * TODO: This is a temporary fix until HIVE-16886 fixes the issue with duplicated IDs + */ + this.notificationHash = notificationHash; + this.pathChange = pathChange.JSONSerialize(); this.createTimeMs = System.currentTimeMillis(); } @@ -91,14 +100,14 @@ public class MSentryPathChange implements MSentryChange { return changeID; } - public long getNotificationID() { - return notificationID; + public String getNotificationHash() { + return notificationHash; } @Override public String toString() { - return "MSentryChange [changeID=" + changeID + " , notificationID= " - + notificationID +" , pathChange= " + pathChange + + return "MSentryChange [changeID=" + changeID + " , notificationHash= " + + notificationHash +" , pathChange= " + pathChange + ", createTime=" + createTimeMs + "]"; } @@ -107,7 +116,7 @@ public class MSentryPathChange implements MSentryChange { final int prime = 31; int result = 1; result = prime * result + Long.valueOf(changeID).hashCode(); - result = prime * result + Long.valueOf(notificationID).hashCode(); + result = prime * result + notificationHash.hashCode(); result = prime * result + ((pathChange == null) ? 0 : pathChange.hashCode()); return result; } @@ -131,7 +140,7 @@ public class MSentryPathChange implements MSentryChange { return false; } - if (notificationID != other.notificationID) { + if (!notificationHash.equals(other.notificationHash)) { return false; } http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo index 734ea7f..59f6826 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo @@ -295,8 +295,16 @@ - - + + + + @@ -305,9 +313,11 @@ + + http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java index 4fe62bf..849e5f8 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/DeltaTransactionBlock.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import org.apache.sentry.core.common.exception.SentryInvalidInputException; import org.apache.sentry.hdfs.PathsUpdate; import org.apache.sentry.hdfs.PermissionsUpdate; +import org.apache.sentry.hdfs.UniquePathsUpdate; import org.apache.sentry.provider.db.service.model.MSentryHmsNotification; import org.apache.sentry.provider.db.service.model.MSentryPathChange; import org.apache.sentry.provider.db.service.model.MSentryPermChange; @@ -87,9 +88,10 @@ public class DeltaTransactionBlock implements TransactionBlock { if (update instanceof PermissionsUpdate) { long lastChangeID = SentryStore.getLastProcessedChangeIDCore(pm, MSentryPermChange.class); pm.makePersistent(new MSentryPermChange(lastChangeID + 1, (PermissionsUpdate) update)); - } else if (update instanceof PathsUpdate) { + } else if (update instanceof UniquePathsUpdate) { long lastChangeID = SentryStore.getLastProcessedChangeIDCore(pm, MSentryPathChange.class); - pm.makePersistent(new MSentryPathChange(lastChangeID + 1, (PathsUpdate) update)); + String eventHash = ((UniquePathsUpdate) update).getEventHash(); + pm.makePersistent(new MSentryPathChange(lastChangeID + 1, eventHash, (PathsUpdate) update)); // Notification id from PATH_UPDATE entry is made persistent in // SENTRY_LAST_NOTIFICATION_ID table. pm.makePersistent(new MSentryHmsNotification(update.getSeqNum())); http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index d7acaea..593b92f 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -51,6 +51,7 @@ import org.apache.sentry.core.common.exception.SentryUserException; import org.apache.sentry.core.common.utils.SentryConstants; import org.apache.sentry.core.model.db.AccessConstants; import org.apache.sentry.core.model.db.DBModelAuthorizable.AuthorizableType; +import org.apache.sentry.hdfs.UniquePathsUpdate; import org.apache.sentry.provider.db.service.model.MAuthzPathsMapping; import org.apache.sentry.provider.db.service.model.MAuthzPathsSnapshotId; import org.apache.sentry.provider.db.service.model.MSentryChange; @@ -2765,7 +2766,7 @@ public class SentryStore { * @throws Exception */ public void addAuthzPathsMapping(final String authzObj, final Collection paths, - final Update update) throws Exception { + final UniquePathsUpdate update) throws Exception { execute(update, new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects @@ -2812,7 +2813,7 @@ public class SentryStore { * @param update the corresponding path delta update */ public void deleteAuthzPathsMapping(final String authzObj, final Iterable paths, - final Update update) throws Exception { + final UniquePathsUpdate update) throws Exception { execute(update, new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects @@ -2863,7 +2864,7 @@ public class SentryStore { * @param authzObj an authzObj to be deleted * @param update the corresponding path delta update */ - public void deleteAllAuthzPathsMapping(final String authzObj, final Update update) + public void deleteAllAuthzPathsMapping(final String authzObj, final UniquePathsUpdate update) throws Exception { execute(update, new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { @@ -2913,7 +2914,7 @@ public class SentryStore { * @param update the corresponding path delta update */ public void renameAuthzPathsMapping(final String oldObj, final String newObj, - final String oldPath, final String newPath, final Update update) throws Exception { + final String oldPath, final String newPath, final UniquePathsUpdate update) throws Exception { execute(update, new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects @@ -2970,7 +2971,7 @@ public class SentryStore { * @param update the corresponding path delta update */ public void renameAuthzObj(final String oldObj, final String newObj, - final Update update) throws Exception { + final UniquePathsUpdate update) throws Exception { execute(update, new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects @@ -3052,7 +3053,7 @@ public class SentryStore { * @throws Exception */ public void updateAuthzPathsMapping(final String authzObj, final String oldPath, - final String newPath, final Update update) throws Exception { + final String newPath, final UniquePathsUpdate update) throws Exception { execute(update, new TransactionBlock() { public Object execute(PersistenceManager pm) throws Exception { pm.setDetachAllOnCommit(false); // No need to detach objects http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java index 9e28da4..713fc3d 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/NotificationProcessor.java @@ -41,6 +41,7 @@ import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; import org.apache.sentry.hdfs.PathsUpdate; import org.apache.sentry.hdfs.PermissionsUpdate; import org.apache.sentry.hdfs.SentryMalformedPathException; +import org.apache.sentry.hdfs.UniquePathsUpdate; import org.apache.sentry.hdfs.Updateable.Update; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.provider.db.service.persistent.SentryStore; @@ -250,7 +251,7 @@ final class NotificationProcessor { if (hdfsSyncEnabled) { List locations = Collections.singletonList(location); - addPaths(dbName, locations, event.getEventId()); + addPaths(dbName, locations, event); return true; } @@ -280,7 +281,7 @@ final class NotificationProcessor { if (hdfsSyncEnabled) { List locations = Collections.singletonList(location); - removePaths(dbName, locations, event.getEventId()); + removePaths(dbName, locations, event); return true; } return false; @@ -315,7 +316,7 @@ final class NotificationProcessor { if (hdfsSyncEnabled) { String authzObj = SentryServiceUtil.getAuthzObj(dbName, tableName); List locations = Collections.singletonList(location); - addPaths(authzObj, locations, event.getEventId()); + addPaths(authzObj, locations, event); return true; } @@ -348,7 +349,7 @@ final class NotificationProcessor { if (hdfsSyncEnabled) { String authzObj = SentryServiceUtil.getAuthzObj(dbName, tableName); - removeAllPaths(authzObj, event.getEventId()); + removeAllPaths(authzObj, event); return true; } @@ -419,7 +420,7 @@ final class NotificationProcessor { } String oldAuthzObj = oldDbName + "." + oldTableName; String newAuthzObj = newDbName + "." + newTableName; - renameAuthzPath(oldAuthzObj, newAuthzObj, oldLocation, newLocation, event.getEventId()); + renameAuthzPath(oldAuthzObj, newAuthzObj, oldLocation, newLocation, event); return true; } @@ -450,7 +451,7 @@ final class NotificationProcessor { return false; } String authzObj = SentryServiceUtil.getAuthzObj(dbName, tableName); - addPaths(authzObj, locations, event.getEventId()); + addPaths(authzObj, locations, event); return true; } @@ -481,7 +482,7 @@ final class NotificationProcessor { return false; } String authzObj = SentryServiceUtil.getAuthzObj(dbName, tableName); - removePaths(authzObj, locations, event.getEventId()); + removePaths(authzObj, locations, event); return true; } @@ -526,7 +527,7 @@ final class NotificationProcessor { } String oldAuthzObj = dbName + "." + tableName; - renameAuthzPath(oldAuthzObj, oldAuthzObj, oldLocation, newLocation, event.getEventId()); + renameAuthzPath(oldAuthzObj, oldAuthzObj, oldLocation, newLocation, event); return true; } @@ -536,14 +537,14 @@ final class NotificationProcessor { * * @param authzObj the given authzObj * @param locations a set of paths need to be added - * @param seqNum notification event ID + * @param event the NotificationEvent object from where authzObj and locations were obtained */ - private void addPaths(String authzObj, Collection locations, long seqNum) + private void addPaths(String authzObj, Collection locations, NotificationEvent event) throws Exception { // AuthzObj is case insensitive authzObj = authzObj.toLowerCase(); - PathsUpdate update = new PathsUpdate(seqNum, false); + UniquePathsUpdate update = new UniquePathsUpdate(event, false); Collection paths = new HashSet<>(locations.size()); // addPath and persist into Sentry DB. // Skip update if encounter malformed path. @@ -554,13 +555,13 @@ final class NotificationProcessor { + "OP : addPath, " + "authzObj : " + authzObj + ", " + "path : " + location + "] - nothing to add" + ", " - + "notification event ID: " + seqNum + "]"); + + "notification event ID: " + event.getEventId() + "]"); } else { LOGGER.debug("HMS Path Update [" + "OP : addPath, " + "authzObj : " + authzObj + ", " + "path : " + location + ", " - + "notification event ID: " + seqNum + "]"); + + "notification event ID: " + event.getEventId() + "]"); update.newPathChange(authzObj).addToAddPaths(splitPath(pathTree)); paths.add(pathTree); } @@ -574,14 +575,14 @@ final class NotificationProcessor { * * @param authzObj the given authzObj * @param locations a set of paths need to be removed - * @param seqNum notification event ID + * @param event the NotificationEvent object from where authzObj and locations were obtained */ - private void removePaths(String authzObj, Collection locations, long seqNum) + private void removePaths(String authzObj, Collection locations, NotificationEvent event) throws Exception { // AuthzObj is case insensitive authzObj = authzObj.toLowerCase(); - PathsUpdate update = new PathsUpdate(seqNum, false); + UniquePathsUpdate update = new UniquePathsUpdate(event, false); Collection paths = new HashSet<>(locations.size()); for (String location : locations) { String pathTree = getPath(location); @@ -590,13 +591,13 @@ final class NotificationProcessor { + "OP : removePath, " + "authzObj : " + authzObj + ", " + "path : " + location + "] - nothing to remove" + ", " - + "notification event ID: " + seqNum + "]"); + + "notification event ID: " + event.getEventId() + "]"); } else { LOGGER.debug("HMS Path Update [" + "OP : removePath, " + "authzObj : " + authzObj + ", " + "path : " + location + ", " - + "notification event ID: " + seqNum + "]"); + + "notification event ID: " + event.getEventId() + "]"); update.newPathChange(authzObj).addToDelPaths(splitPath(pathTree)); paths.add(pathTree); } @@ -610,9 +611,9 @@ final class NotificationProcessor { * delta path change to Sentry DB. * * @param authzObj the given authzObj to be deleted - * @param seqNum notification event ID + * @param event the NotificationEvent object from where authzObj and locations were obtained */ - private void removeAllPaths(String authzObj, long seqNum) + private void removeAllPaths(String authzObj, NotificationEvent event) throws Exception { // AuthzObj is case insensitive authzObj = authzObj.toLowerCase(); @@ -620,8 +621,8 @@ final class NotificationProcessor { LOGGER.debug("HMS Path Update [" + "OP : removeAllPaths, " + "authzObj : " + authzObj + ", " - + "notification event ID: " + seqNum + "]"); - PathsUpdate update = new PathsUpdate(seqNum, false); + + "notification event ID: " + event.getEventId() + "]"); + UniquePathsUpdate update = new UniquePathsUpdate(event, false); update.newPathChange(authzObj).addToDelPaths( Lists.newArrayList(PathsUpdate.ALL_PATHS)); sentryStore.deleteAllAuthzPathsMapping(authzObj, update); @@ -636,9 +637,10 @@ final class NotificationProcessor { * @param newAuthzObj the new name to be changed to * @param oldLocation a existing path of the given authzObj * @param newLocation a new path to be changed to + * @param event the NotificationEvent object from where authzObj and locations were obtained */ private void renameAuthzPath(String oldAuthzObj, String newAuthzObj, String oldLocation, - String newLocation, long seqNum) throws Exception { + String newLocation, NotificationEvent event) throws Exception { // AuthzObj is case insensitive oldAuthzObj = oldAuthzObj.toLowerCase(); newAuthzObj = newAuthzObj.toLowerCase(); @@ -651,13 +653,13 @@ final class NotificationProcessor { + "newAuthzObj : " + newAuthzObj + ", " + "oldLocation : " + oldLocation + ", " + "newLocation : " + newLocation + ", " - + "notification event ID: " + seqNum + "]"); + + "notification event ID: " + event.getEventId() + "]"); // In the case of HiveObj name has changed if (!oldAuthzObj.equalsIgnoreCase(newAuthzObj)) { // Skip update if encounter malformed path for both oldLocation and newLocation. if ((oldPathTree != null) && (newPathTree != null)) { - PathsUpdate update = new PathsUpdate(seqNum, false); + UniquePathsUpdate update = new UniquePathsUpdate(event, false); update.newPathChange(oldAuthzObj).addToDelPaths(splitPath(oldPathTree)); update.newPathChange(newAuthzObj).addToAddPaths(splitPath(newPathTree)); if (oldLocation.equals(newLocation)) { @@ -671,18 +673,18 @@ final class NotificationProcessor { newPathTree, update); } } else { - updateAuthzPathsMapping(oldAuthzObj, oldPathTree, newAuthzObj, newPathTree, seqNum); + updateAuthzPathsMapping(oldAuthzObj, oldPathTree, newAuthzObj, newPathTree, event); } } else if (!oldLocation.equals(newLocation)) { // Only Location has changed, e.g. Alter table set location if ((oldPathTree != null) && (newPathTree != null)) { - PathsUpdate update = new PathsUpdate(seqNum, false); + UniquePathsUpdate update = new UniquePathsUpdate(event, false); update.newPathChange(oldAuthzObj).addToDelPaths(splitPath(oldPathTree)); update.newPathChange(oldAuthzObj).addToAddPaths(splitPath(newPathTree)); sentryStore.updateAuthzPathsMapping(oldAuthzObj, oldPathTree, newPathTree, update); } else { - updateAuthzPathsMapping(oldAuthzObj, oldPathTree, newAuthzObj, newPathTree, seqNum); + updateAuthzPathsMapping(oldAuthzObj, oldPathTree, newAuthzObj, newPathTree,event); } } else { LOGGER.error("Update Notification for Auhorizable object {}, with no change, skipping", @@ -693,15 +695,15 @@ final class NotificationProcessor { } private void updateAuthzPathsMapping(String oldAuthzObj, String oldPathTree, - String newAuthzObj, String newPathTree, long seqNum) throws Exception { + String newAuthzObj, String newPathTree, NotificationEvent event) throws Exception { if (oldPathTree != null) { - PathsUpdate update = new PathsUpdate(seqNum, false); + UniquePathsUpdate update = new UniquePathsUpdate(event, false); update.newPathChange(oldAuthzObj).addToDelPaths(splitPath(oldPathTree)); sentryStore.deleteAuthzPathsMapping(oldAuthzObj, Collections.singleton(oldPathTree), update); } else if (newPathTree != null) { - PathsUpdate update = new PathsUpdate(seqNum, false); + UniquePathsUpdate update = new UniquePathsUpdate(event, false); update.newPathChange(newAuthzObj).addToAddPaths(splitPath(newPathTree)); sentryStore.addAuthzPathsMapping(newAuthzObj, Collections.singleton(newPathTree), http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.derby.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.derby.sql b/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.derby.sql index 019b040..49b7959 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.derby.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.derby.sql @@ -12,7 +12,7 @@ ALTER TABLE SENTRY_PERM_CHANGE ADD CONSTRAINT SENTRY_PERM_CHANGE_PK PRIMARY KEY CREATE TABLE SENTRY_PATH_CHANGE ( CHANGE_ID BIGINT NOT NULL, - NOTIFICATION_ID BIGINT NOT NULL, + NOTIFICATION_HASH CHAR(40) NOT NULL, CREATE_TIME_MS BIGINT NOT NULL, PATH_CHANGE CLOB NOT NULL ); @@ -20,9 +20,12 @@ CREATE TABLE SENTRY_PATH_CHANGE -- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] ALTER TABLE SENTRY_PATH_CHANGE ADD CONSTRAINT SENTRY_PATH_CHANGE_PK PRIMARY KEY (CHANGE_ID); +CREATE UNIQUE INDEX NOTIFICATION_HASH_INDEX ON SENTRY_PATH_CHANGE (NOTIFICATION_HASH); -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE SENTRY_HMS_NOTIFICATION_ID ( NOTIFICATION_ID BIGINT NOT NULL ); + +CREATE INDEX SENTRY_HMS_NOTIFICATION_ID_INDEX ON SENTRY_HMS_NOTIFICATION_ID (NOTIFICATION_ID); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.mysql.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.mysql.sql b/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.mysql.sql index a0223dc..0bda5cd 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.mysql.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.mysql.sql @@ -11,14 +11,19 @@ CREATE TABLE `SENTRY_PERM_CHANGE` CREATE TABLE `SENTRY_PATH_CHANGE` ( `CHANGE_ID` BIGINT NOT NULL, - `NOTIFICATION_ID` BIGINT NOT NULL, + `NOTIFICATION_HASH` CHAR(40) NOT NULL, `CREATE_TIME_MS` BIGINT NOT NULL, `PATH_CHANGE` longtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, CONSTRAINT `SENTRY_PATH_CHANGE_PK` PRIMARY KEY (`CHANGE_ID`) ) ENGINE=INNODB; +-- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] +CREATE UNIQUE INDEX `NOTIFICATION_HASH_INDEX` ON `SENTRY_PATH_CHANGE` (`NOTIFICATION_HASH`); + -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE `SENTRY_HMS_NOTIFICATION_ID` ( `NOTIFICATION_ID` BIGINT NOT NULL -)ENGINE=INNODB; \ No newline at end of file +)ENGINE=INNODB; + +CREATE INDEX `SENTRY_HMS_NOTIFICATION_ID_INDEX` ON `SENTRY_HMS_NOTIFICATION_ID` (`NOTIFICATION_ID`); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.oracle.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.oracle.sql b/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.oracle.sql index 733cebf..8dd8316 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.oracle.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.oracle.sql @@ -12,16 +12,19 @@ ALTER TABLE "SENTRY_PERM_CHANGE" ADD CONSTRAINT "SENTRY_PERM_CHANGE_PK" PRIMARY CREATE TABLE "SENTRY_PATH_CHANGE" ( "CHANGE_ID" NUMBER NOT NULL, - "NOTIFICATION_ID" NUMBER NOT NULL, + "NOTIFICATION_HASH" CHAR(40) NOT NULL, "CREATE_TIME_MS" NUMBER NOT NULL, "PATH_CHANGE" CLOB NOT NULL ); -- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] +CREATE UNIQUE INDEX "NOTIFICATION_HASH_INDEX" ON "SENTRY_PATH_CHANGE" ("NOTIFICATION_HASH"); ALTER TABLE "SENTRY_PATH_CHANGE" ADD CONSTRAINT SENTRY_PATH_CHANGE_PK PRIMARY KEY ("CHANGE_ID"); -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE "SENTRY_HMS_NOTIFICATION_ID" ( "NOTIFICATION_ID" NUMBER NOT NULL -); \ No newline at end of file +); + +CREATE INDEX "SENTRY_HMS_NOTIFICATION_ID_INDEX" ON "SENTRY_HMS_NOTIFICATION_ID" ("NOTIFICATION_ID"); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.postgres.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.postgres.sql b/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.postgres.sql index cd7afd4..2610d6c 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.postgres.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/008-SENTRY-1569.postgres.sql @@ -11,14 +11,19 @@ CREATE TABLE "SENTRY_PERM_CHANGE" CREATE TABLE "SENTRY_PATH_CHANGE" ( "CHANGE_ID" bigint NOT NULL, - "NOTIFICATION_ID" bigint NOT NULL, + "NOTIFICATION_HASH" char(40) NOT NULL, "CREATE_TIME_MS" bigint NOT NULL, "PATH_CHANGE" text NOT NULL, CONSTRAINT "SENTRY_PATH_CHANGE_PK" PRIMARY KEY ("CHANGE_ID") ); +-- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] +CREATE UNIQUE INDEX "NOTIFICATION_HASH_INDEX" ON "SENTRY_PATH_CHANGE" ("NOTIFICATION_HASH"); + -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE "SENTRY_HMS_NOTIFICATION_ID" ( "NOTIFICATION_ID" bigint NOT NULL ); + +CREATE INDEX "SENTRY_HMS_NOTIFICATION_ID_INDEX" ON "SENTRY_HMS_NOTIFICATION_ID" ("NOTIFICATION_ID"); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/sentry-db2-2.0.0.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/sentry-db2-2.0.0.sql b/sentry-provider/sentry-provider-db/src/main/resources/sentry-db2-2.0.0.sql index 44124c6..93d6e70 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/sentry-db2-2.0.0.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/sentry-db2-2.0.0.sql @@ -234,16 +234,19 @@ ALTER TABLE SENTRY_PERM_CHANGE ADD CONSTRAINT SENTRY_PERM_CHANGE_PK PRIMARY KEY CREATE TABLE SENTRY_PATH_CHANGE ( CHANGE_ID bigint NOT NULL, - NOTIFICATION_ID bigint NOT NULL, + NOTIFICATION_HASH char(40) NOT NULL, CREATE_TIME_MS bigint NOT NULL, PATH_CHANGE CLOB NOT NULL ); -- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] ALTER TABLE SENTRY_PATH_CHANGE ADD CONSTRAINT SENTRY_PATH_CHANGE_PK PRIMARY KEY (CHANGE_ID); +CREATE UNIQUE INDEX NOTIFICATION_HASH_INDEX ON SENTRY_PATH_CHANGE (NOTIFICATION_HASH); -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE SENTRY_HMS_NOTIFICATION_ID ( NOTIFICATION_ID bigint NOT NULL -); \ No newline at end of file +); + +CREATE INDEX SENTRY_HMS_NOTIFICATION_ID_INDEX ON SENTRY_HMS_NOTIFICATION_ID (NOTIFICATION_ID); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/sentry-derby-2.0.0.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/sentry-derby-2.0.0.sql b/sentry-provider/sentry-provider-db/src/main/resources/sentry-derby-2.0.0.sql index 3a7b46e..b4e39fd 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/sentry-derby-2.0.0.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/sentry-derby-2.0.0.sql @@ -234,17 +234,19 @@ ALTER TABLE SENTRY_PERM_CHANGE ADD CONSTRAINT SENTRY_PERM_CHANGE_PK PRIMARY KEY CREATE TABLE SENTRY_PATH_CHANGE ( CHANGE_ID BIGINT NOT NULL, - NOTIFICATION_ID BIGINT NOT NULL, + NOTIFICATION_HASH CHAR(40) NOT NULL, CREATE_TIME_MS BIGINT NOT NULL, PATH_CHANGE CLOB NOT NULL ); -- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] ALTER TABLE SENTRY_PATH_CHANGE ADD CONSTRAINT SENTRY_PATH_CHANGE_PK PRIMARY KEY (CHANGE_ID); - +CREATE UNIQUE INDEX NOTIFICATION_HASH_INDEX ON SENTRY_PATH_CHANGE (NOTIFICATION_HASH); -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE SENTRY_HMS_NOTIFICATION_ID ( NOTIFICATION_ID BIGINT NOT NULL ); + +CREATE INDEX SENTRY_HMS_NOTIFICATION_ID_INDEX ON SENTRY_HMS_NOTIFICATION_ID (NOTIFICATION_ID); http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/sentry-mysql-2.0.0.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/sentry-mysql-2.0.0.sql b/sentry-provider/sentry-provider-db/src/main/resources/sentry-mysql-2.0.0.sql index 7475a5d..4326a38 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/sentry-mysql-2.0.0.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/sentry-mysql-2.0.0.sql @@ -269,14 +269,19 @@ CREATE TABLE `SENTRY_PERM_CHANGE` CREATE TABLE `SENTRY_PATH_CHANGE` ( `CHANGE_ID` BIGINT NOT NULL, - `NOTIFICATION_ID` BIGINT NOT NULL, + `NOTIFICATION_HASH` CHAR(40) NOT NULL, `CREATE_TIME_MS` BIGINT NOT NULL, `PATH_CHANGE` longtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, CONSTRAINT `SENTRY_PATH_CHANGE_PK` PRIMARY KEY (`CHANGE_ID`) ) ENGINE=INNODB; +-- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] +CREATE UNIQUE INDEX `NOTIFICATION_HASH_INDEX` ON `SENTRY_PATH_CHANGE` (`NOTIFICATION_HASH`); + -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE `SENTRY_HMS_NOTIFICATION_ID` ( `NOTIFICATION_ID` BIGINT NOT NULL )ENGINE=INNODB; + +CREATE INDEX `SENTRY_HMS_NOTIFICATION_ID_INDEX` ON `SENTRY_HMS_NOTIFICATION_ID` (`NOTIFICATION_ID`); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/sentry-oracle-2.0.0.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/sentry-oracle-2.0.0.sql b/sentry-provider/sentry-provider-db/src/main/resources/sentry-oracle-2.0.0.sql index 20c4cd9..92676a5 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/sentry-oracle-2.0.0.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/sentry-oracle-2.0.0.sql @@ -250,16 +250,19 @@ ALTER TABLE "SENTRY_PERM_CHANGE" ADD CONSTRAINT "SENTRY_PERM_CHANGE_PK" PRIMARY CREATE TABLE "SENTRY_PATH_CHANGE" ( "CHANGE_ID" NUMBER NOT NULL, - "NOTIFICATION_ID" NUMBER NOT NULL, + "NOTIFICATION_HASH" CHAR(40) NOT NULL, "CREATE_TIME_MS" NUMBER NOT NULL, "PATH_CHANGE" CLOB NOT NULL ); -- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] +CREATE UNIQUE INDEX "NOTIFICATION_HASH_INDEX" ON "SENTRY_PATH_CHANGE" ("NOTIFICATION_HASH"); ALTER TABLE "SENTRY_PATH_CHANGE" ADD CONSTRAINT SENTRY_PATH_CHANGE_PK PRIMARY KEY ("CHANGE_ID"); -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE "SENTRY_HMS_NOTIFICATION_ID" ( "NOTIFICATION_ID" NUMBER NOT NULL -); \ No newline at end of file +); + +CREATE INDEX "SENTRY_HMS_NOTIFICATION_ID_INDEX" ON "SENTRY_HMS_NOTIFICATION_ID" ("NOTIFICATION_ID"); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/sentry-postgres-2.0.0.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/sentry-postgres-2.0.0.sql b/sentry-provider/sentry-provider-db/src/main/resources/sentry-postgres-2.0.0.sql index 794ff17..05eb751 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/sentry-postgres-2.0.0.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/sentry-postgres-2.0.0.sql @@ -257,16 +257,21 @@ CREATE TABLE "SENTRY_PERM_CHANGE" CREATE TABLE "SENTRY_PATH_CHANGE" ( "CHANGE_ID" bigint NOT NULL, - "NOTIFICATION_ID" bigint NOT NULL, + "NOTIFICATION_HASH" CHAR(40) NOT NULL, "CREATE_TIME_MS" bigint NOT NULL, "PATH_CHANGE" text NOT NULL, CONSTRAINT "SENTRY_PATH_CHANGE_PK" PRIMARY KEY ("CHANGE_ID") ); +-- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] +CREATE UNIQUE INDEX "NOTIFICATION_HASH_INDEX" ON "SENTRY_PATH_CHANGE" ("NOTIFICATION_HASH"); + -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE "SENTRY_HMS_NOTIFICATION_ID" ( "NOTIFICATION_ID" bigint NOT NULL ); +CREATE INDEX "SENTRY_HMS_NOTIFICATION_ID_INDEX" ON "SENTRY_HMS_NOTIFICATION_ID" ("NOTIFICATION_ID"); + COMMIT; http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/main/resources/sentry-upgrade-db2-1.7.0-to-2.0.0.sql ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/resources/sentry-upgrade-db2-1.7.0-to-2.0.0.sql b/sentry-provider/sentry-provider-db/src/main/resources/sentry-upgrade-db2-1.7.0-to-2.0.0.sql index 065921e..62856de 100644 --- a/sentry-provider/sentry-provider-db/src/main/resources/sentry-upgrade-db2-1.7.0-to-2.0.0.sql +++ b/sentry-provider/sentry-provider-db/src/main/resources/sentry-upgrade-db2-1.7.0-to-2.0.0.sql @@ -81,13 +81,14 @@ ALTER TABLE "SENTRY_PERM_CHANGE" ADD CONSTRAINT "SENTRY_PERM_CHANGE_PK" PRIMARY CREATE TABLE SENTRY_PATH_CHANGE ( CHANGE_ID bigint NOT NULL, - NOTIFICATION_ID bigint NOT NULL, + NOTIFICATION_HASH CHAR(40) NOT NULL, CREATE_TIME_MS bigint NOT NULL, PATH_CHANGE CLOB NOT NULL ); -- Constraints for table SENTRY_PATH_CHANGE for class [org.apache.sentry.provider.db.service.model.MSentryPathChange] ALTER TABLE SENTRY_PATH_CHANGE ADD CONSTRAINT SENTRY_PATH_CHANGE_PK PRIMARY KEY (CHANGE_ID); +CREATE UNIQUE INDEX NOTIFICATION_HASH_INDEX ON SENTRY_PATH_CHANGE (NOTIFICATION_HASH); -- Table SENTRY_HMS_NOTIFICATION_ID for classes [org.apache.sentry.provider.db.service.model.MSentryHmsNotification] CREATE TABLE SENTRY_HMS_NOTIFICATION_ID @@ -95,5 +96,7 @@ CREATE TABLE SENTRY_HMS_NOTIFICATION_ID NOTIFICATION_ID bigint NOT NULL ); +CREATE INDEX SENTRY_HMS_NOTIFICATION_ID_INDEX ON SENTRY_HMS_NOTIFICATION_ID (NOTIFICATION_ID); + -- Version update UPDATE SENTRY_VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Sentry release version 2.0.0' WHERE VER_ID=1; http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/model/TestMSentryUtil.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/model/TestMSentryUtil.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/model/TestMSentryUtil.java index 6ff79f8..34aa465 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/model/TestMSentryUtil.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/model/TestMSentryUtil.java @@ -37,32 +37,32 @@ public class TestMSentryUtil { List changes = new ArrayList<>(); PathsUpdate update = new PathsUpdate(1, false); - changes.add(new MSentryPathChange(1, update)); + changes.add(new MSentryPathChange(1, "u1", update)); assertEquals("Collapsed string should match", "[1]", MSentryUtil.collapseChangeIDsToString(changes)); assertTrue("List of changes should be consecutive", MSentryUtil.isConsecutive(changes)); - changes.add(new MSentryPathChange(2, update)); + changes.add(new MSentryPathChange(2, "u2",update)); assertEquals("Collapsed string should match", "[1, 2]", MSentryUtil.collapseChangeIDsToString(changes)); assertTrue("List of changes should be consecutive", MSentryUtil.isConsecutive(changes)); - changes.add(new MSentryPathChange(4, update)); + changes.add(new MSentryPathChange(4, "u4",update)); assertEquals("Collapsed string should match", "[1, 2, 4]", MSentryUtil.collapseChangeIDsToString(changes)); assertFalse("List of changes should not be consecutive", MSentryUtil.isConsecutive(changes)); - changes.add(new MSentryPathChange(5, update)); + changes.add(new MSentryPathChange(5, "u5",update)); assertEquals("Collapsed string should match", "[1, 2, 4, 5]", MSentryUtil.collapseChangeIDsToString(changes)); assertFalse("List of changes should not be consecutive", MSentryUtil.isConsecutive(changes)); - changes.add(new MSentryPathChange(6, update)); + changes.add(new MSentryPathChange(6, "u6",update)); assertEquals("Collapsed string should match", "[1, 2, 4-6]", MSentryUtil.collapseChangeIDsToString(changes)); assertFalse("List of changes should not be consecutive", MSentryUtil.isConsecutive(changes)); - changes.add(new MSentryPathChange(8, update)); + changes.add(new MSentryPathChange(8, "u8",update)); assertEquals("Collapsed string should match", "[1, 2, 4-6, 8]", MSentryUtil.collapseChangeIDsToString(changes)); assertFalse("List of changes should not be consecutive", MSentryUtil.isConsecutive(changes)); http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index 8f60b58..3da6a4e 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -47,6 +47,7 @@ import org.apache.sentry.core.common.exception.SentryGrantDeniedException; import org.apache.sentry.core.common.exception.SentryNoSuchObjectException; import org.apache.sentry.hdfs.PathsUpdate; import org.apache.sentry.hdfs.PermissionsUpdate; +import org.apache.sentry.hdfs.UniquePathsUpdate; import org.apache.sentry.hdfs.Updateable; import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges; import org.apache.sentry.hdfs.service.thrift.TRoleChanges; @@ -2485,8 +2486,8 @@ public class TestSentryStore extends org.junit.Assert { sentryStore.persistFullPathsImage(new HashMap>(), 0); // Create two path updates with the same sequence ID - PathsUpdate update1 = new PathsUpdate(notificationID, false); - PathsUpdate update2 = new PathsUpdate(notificationID, false); + UniquePathsUpdate update1 = new UniquePathsUpdate("u1", notificationID, false); + UniquePathsUpdate update2 = new UniquePathsUpdate("u2", notificationID, false); // Populate the path updates with different objects and paths update1.newPathChange("db1").addToAddPaths(Arrays.asList("/hive/db1")); @@ -2513,11 +2514,13 @@ public class TestSentryStore extends org.junit.Assert { List pathsChanges = sentryStore.getMSentryPathChanges(); assertEquals(2, pathsChanges.size()); assertEquals(1, pathsChanges.get(0).getChangeID()); // changeID = 1 - assertEquals(notificationID, pathsChanges.get(0).getNotificationID()); assertTrue(pathsChanges.get(0).getPathChange().contains("/hive/db1")); assertEquals(2, pathsChanges.get(1).getChangeID()); // changeID = 2 - assertEquals(notificationID, pathsChanges.get(1).getNotificationID()); assertTrue(pathsChanges.get(1).getPathChange().contains("/hive/db2")); + + // Check that the SHA1 hash calculated for unique notifications is correct + assertEquals("u1", pathsChanges.get(0).getNotificationHash()); + assertEquals("u2", pathsChanges.get(1).getNotificationHash()); } @Test @@ -2542,7 +2545,7 @@ public class TestSentryStore extends org.junit.Assert { // Add "db1.table1" authzObj Long lastNotificationId = sentryStore.getLastProcessedNotificationID(); - PathsUpdate addUpdate = new PathsUpdate(1, false); + UniquePathsUpdate addUpdate = new UniquePathsUpdate("u1", 1, false); addUpdate.newPathChange("db1.table"). addToAddPaths(Arrays.asList("db1", "tbl1")); addUpdate.newPathChange("db1.table"). @@ -2564,7 +2567,7 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(1, lastNotificationId.longValue()); // Delete path 'db1.db/tbl1' from "db1.table1" authzObj. - PathsUpdate delUpdate = new PathsUpdate(2, false); + UniquePathsUpdate delUpdate = new UniquePathsUpdate("u2",2, false); delUpdate.newPathChange("db1.table") .addToDelPaths(Arrays.asList("db1", "tbl1")); sentryStore.deleteAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1"), delUpdate); @@ -2581,7 +2584,7 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(2, lastNotificationId.longValue()); // Delete "db1.table" authzObj from the authzObj -> [Paths] mapping. - PathsUpdate delAllupdate = new PathsUpdate(3, false); + UniquePathsUpdate delAllupdate = new UniquePathsUpdate("u3",3, false); delAllupdate.newPathChange("db1.table") .addToDelPaths(Lists.newArrayList(PathsUpdate.ALL_PATHS)); sentryStore.deleteAllAuthzPathsMapping("db1.table", delAllupdate); @@ -2612,7 +2615,7 @@ public class TestSentryStore extends org.junit.Assert { // Rename path of 'db1.table1' from 'db1.table1' to 'db1.newTable1' - PathsUpdate renameUpdate = new PathsUpdate(1, false); + UniquePathsUpdate renameUpdate = new UniquePathsUpdate("u1",1, false); renameUpdate.newPathChange("db1.table1") .addToDelPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "table1")); renameUpdate.newPathChange("db1.newTable1") @@ -2634,7 +2637,7 @@ public class TestSentryStore extends org.junit.Assert { lastNotificationId = sentryStore.getLastProcessedNotificationID(); assertEquals(1, lastNotificationId.longValue()); // Rename 'db1.table1' to "db1.table2" but did not change its location. - renameUpdate = new PathsUpdate(2, false); + renameUpdate = new UniquePathsUpdate("u2",2, false); renameUpdate.newPathChange("db1.newTable1") .addToDelPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1")); renameUpdate.newPathChange("db1.newTable2") @@ -2656,7 +2659,7 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(renameUpdate.JSONSerialize(), renamePathChange.getPathChange()); // Update path of 'db1.newTable2' from 'db1.newTable1' to 'db1.newTable2' - PathsUpdate update = new PathsUpdate(3, false); + UniquePathsUpdate update = new UniquePathsUpdate("u3",3, false); update.newPathChange("db1.newTable1") .addToDelPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1")); update.newPathChange("db1.newTable1") @@ -2735,7 +2738,7 @@ public class TestSentryStore extends org.junit.Assert { long notificationID = 1; // Add some paths first (these should be replaced) - PathsUpdate addUpdate = new PathsUpdate(notificationID, false); + UniquePathsUpdate addUpdate = new UniquePathsUpdate("u1", notificationID, false); addUpdate.newPathChange("db1.table").addToAddPaths(Arrays.asList("db1", "tbl1")); addUpdate.newPathChange("db1.table").addToAddPaths(Arrays.asList("db1", "tbl2")); sentryStore.addAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1", "db1/tbl2"), addUpdate); @@ -2749,7 +2752,7 @@ public class TestSentryStore extends org.junit.Assert { // Add new paths notificationID ++; - PathsUpdate newAddUpdate = new PathsUpdate(notificationID, false); + UniquePathsUpdate newAddUpdate = new UniquePathsUpdate("u2", notificationID, false); newAddUpdate.newPathChange("db2.table").addToAddPaths(Arrays.asList("db2", "tbl1")); newAddUpdate.newPathChange("db2.table").addToAddPaths(Arrays.asList("db2", "tbl2")); sentryStore.addAuthzPathsMapping("db2.table", Sets.newHashSet("db2/tbl1", "db2/tbl2"), newAddUpdate); @@ -2761,7 +2764,7 @@ public class TestSentryStore extends org.junit.Assert { // Delete one path notificationID ++; - PathsUpdate delUpdate = new PathsUpdate(notificationID, false); + UniquePathsUpdate delUpdate = new UniquePathsUpdate("u3", notificationID, false); delUpdate.newPathChange("db2.table").addToDelPaths(Arrays.asList("db2", "tbl1")); sentryStore.deleteAuthzPathsMapping("db2.table", Sets.newHashSet("db2/tbl1"), delUpdate); pathsImage = sentryStore.retrieveFullPathsImage(); @@ -2797,7 +2800,7 @@ public class TestSentryStore extends org.junit.Assert { // Rename path of 'db1.table1' from 'db1.table1' to 'db1.newTable1' notificationID ++; - PathsUpdate renameUpdate = new PathsUpdate(notificationID, false); + UniquePathsUpdate renameUpdate = new UniquePathsUpdate("u1", notificationID, false); renameUpdate.newPathChange("db3.table1") .addToDelPaths(Arrays.asList("another-warehouse", "db3.db", "table1.1")); renameUpdate.newPathChange("db1.newTable1") @@ -2814,7 +2817,7 @@ public class TestSentryStore extends org.junit.Assert { // Update path of 'db1.newTable2' from 'db1.newTable1' to 'db1.newTable2' notificationID++; - PathsUpdate update = new PathsUpdate(notificationID, false); + UniquePathsUpdate update = new UniquePathsUpdate("u2", notificationID, false); update.newPathChange("db1.newTable1") .addToDelPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "newTable1")); update.newPathChange("db1.newTable1") @@ -3273,7 +3276,7 @@ public class TestSentryStore extends org.junit.Assert { // Rename path of 'db1.table1' from 'db1.table1' to 'db1.newTable1' lastNotificationId ++; - PathsUpdate renameUpdate = new PathsUpdate(lastNotificationId, false); + UniquePathsUpdate renameUpdate = new UniquePathsUpdate("u1", lastNotificationId, false); renameUpdate.newPathChange("db1.table1") .addToDelPaths(Arrays.asList("user", "hive", "warehouse", "db1.db", "table1")); renameUpdate.newPathChange("db1.newTable1") @@ -3310,7 +3313,7 @@ public class TestSentryStore extends org.junit.Assert { @Test public void testIsAuthzPathsMappingEmpty() throws Exception { // Add "db1.table1" authzObj - PathsUpdate addUpdate = new PathsUpdate(1, false); + UniquePathsUpdate addUpdate = new UniquePathsUpdate("u1",1, false); addUpdate.newPathChange("db1.table"). addToAddPaths(Arrays.asList("db1", "tbl1")); addUpdate.newPathChange("db1.table"). @@ -3345,7 +3348,7 @@ public class TestSentryStore extends org.junit.Assert { // Add "db1.table1" authzObj Long lastNotificationId = sentryStore.getLastProcessedNotificationID(); - PathsUpdate addUpdate = new PathsUpdate(1, false); + UniquePathsUpdate addUpdate = new UniquePathsUpdate("u1",1, false); addUpdate.newPathChange("db1.table"). addToAddPaths(Arrays.asList("db1", "tbl1")); addUpdate.newPathChange("db1.table"). @@ -3364,7 +3367,7 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(0, lastChangeID); // Delete path 'db1.db/tbl1' from "db1.table1" authzObj. - PathsUpdate delUpdate = new PathsUpdate(2, false); + UniquePathsUpdate delUpdate = new UniquePathsUpdate("u2",2, false); delUpdate.newPathChange("db1.table") .addToDelPaths(Arrays.asList("db1", "tbl1")); localSentryStore.deleteAuthzPathsMapping("db1.table", Sets.newHashSet("db1/tbl1"), delUpdate); @@ -3378,7 +3381,7 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(0, lastChangeID); // Delete "db1.table" authzObj from the authzObj -> [Paths] mapping. - PathsUpdate delAllupdate = new PathsUpdate(3, false); + UniquePathsUpdate delAllupdate = new UniquePathsUpdate("u3",3, false); delAllupdate.newPathChange("db1.table") .addToDelPaths(Lists.newArrayList(PathsUpdate.ALL_PATHS)); localSentryStore.deleteAllAuthzPathsMapping("db1.table", delAllupdate); http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java index c5fedfc..f56384a 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java @@ -43,7 +43,7 @@ import org.apache.hive.hcatalog.messaging.HCatEventMessage.EventType; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars; import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory; -import org.apache.sentry.hdfs.Updateable; +import org.apache.sentry.hdfs.UniquePathsUpdate; import org.apache.sentry.provider.db.service.persistent.PathsImage; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; @@ -476,7 +476,7 @@ public class TestHMSFollower { Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong()); //noinspection unchecked Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); Configuration configuration = new Configuration(); HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, null, @@ -501,7 +501,7 @@ public class TestHMSFollower { // and persistLastProcessedNotificationID was not invoked. //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong()); reset(sentryStore); events.clear(); @@ -525,7 +525,7 @@ public class TestHMSFollower { // and persistLastProcessedNotificationID was not invoked. //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong()); reset(sentryStore); events.clear(); @@ -562,7 +562,7 @@ public class TestHMSFollower { // Make sure that updateAuthzPathsMapping was invoked once to handle ALTER_PARTITION // notification and persistLastProcessedNotificationID was not invoked. verify(sentryStore, times(1)).updateAuthzPathsMapping(Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class)); + Mockito.anyString(), Mockito.anyString(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(0)).persistLastProcessedNotificationID(inputEventId - 1); reset(sentryStore); events.clear(); @@ -586,7 +586,7 @@ public class TestHMSFollower { // and persistLastProcessedNotificationID was not invoked. //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong()); } @@ -611,7 +611,7 @@ public class TestHMSFollower { Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong()); //noinspection unchecked Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); Configuration configuration = new Configuration(); // enable HDFS sync, so perm and path changes will be saved into DB @@ -641,7 +641,7 @@ public class TestHMSFollower { // and persistLastProcessedNotificationID was not invoked. //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong()); reset(sentryStore); events.clear(); @@ -666,10 +666,10 @@ public class TestHMSFollower { // to handle CREATE_TABLE notification // and persistLastProcessedNotificationID is explicitly invoked verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(), - Mockito.any(Updateable.Update.class)); + Mockito.any(UniquePathsUpdate.class)); //noinspection unchecked verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(1)).persistLastProcessedNotificationID(Mockito.anyLong()); reset(sentryStore); events.clear(); @@ -693,7 +693,7 @@ public class TestHMSFollower { // and persistLastProcessedNotificationID was not invoked. //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(0)).persistLastProcessedNotificationID(Mockito.anyLong()); } @@ -715,7 +715,7 @@ public class TestHMSFollower { //noinspection unchecked Mockito.doNothing().when(sentryStore) .addAuthzPathsMapping(Mockito.anyString(), Mockito.anyCollection(), - Mockito.any(Updateable.Update.class)); + Mockito.any(UniquePathsUpdate.class)); // Create invalid notification event. The location of the storage descriptor is null, which is invalid for creating table StorageDescriptor invalidSd = new StorageDescriptor(); @@ -750,7 +750,7 @@ public class TestHMSFollower { // next valid event update path, which updates notification ID //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), Mockito.anyCollection(), - Mockito.any(Updateable.Update.class)); + Mockito.any(UniquePathsUpdate.class)); } @Test http://git-wip-us.apache.org/repos/asf/sentry/blob/ab906afa/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java index 0fd4ff0..00f9b39 100644 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory; -import org.apache.sentry.hdfs.Updateable; +import org.apache.sentry.hdfs.UniquePathsUpdate; import org.apache.sentry.provider.db.service.persistent.SentryStore; import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; import org.junit.After; @@ -97,7 +97,7 @@ public class TestNotificationProcessor { authorizable.setDb("db1"); //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(1)).dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable)); @@ -119,7 +119,7 @@ public class TestNotificationProcessor { //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); //making sure that privileges are not dropped verify(sentryStore, times(0)).dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable)); @@ -153,7 +153,7 @@ public class TestNotificationProcessor { //noinspection unchecked verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(1)).dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable)); reset(sentryStore); @@ -175,7 +175,7 @@ public class TestNotificationProcessor { //noinspection unchecked verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(0)).dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable)); } @@ -211,7 +211,7 @@ public class TestNotificationProcessor { //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(1)).dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable)); @@ -239,7 +239,7 @@ public class TestNotificationProcessor { //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); // Making sure that privileges are not dropped verify(sentryStore, times(0)).dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable)); @@ -280,7 +280,7 @@ public class TestNotificationProcessor { authorizable.setTable(tableName); verify(sentryStore, times(1)).deleteAllAuthzPathsMapping(Mockito.anyString(), - Mockito.any(Updateable.Update.class)); + Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(1)).dropPrivilege(authorizable, NotificationProcessor.getPermUpdatableOnDrop(authorizable)); @@ -331,7 +331,7 @@ public class TestNotificationProcessor { newAuthorizable.setTable(newTableName); verify(sentryStore, times(1)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(), - Mockito.any(Updateable.Update.class)); + Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable, NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable)); @@ -384,7 +384,7 @@ public class TestNotificationProcessor { newAuthorizable.setTable(newTableName); verify(sentryStore, times(1)).renameAuthzPathsMapping(Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class)); + Mockito.anyString(), Mockito.anyString(), Mockito.any(UniquePathsUpdate.class)); verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable, NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable)); @@ -406,7 +406,7 @@ public class TestNotificationProcessor { Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong()); //noinspection unchecked Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); Configuration authConf = new Configuration(); // enable HDFS sync, so perm and path changes will be saved into DB @@ -435,7 +435,7 @@ public class TestNotificationProcessor { // and persistLastProcessedNotificationID was not invoked. //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); reset(sentryStore); // Create alter table notification with out actually changing anything. @@ -456,10 +456,10 @@ public class TestNotificationProcessor { // to handle CREATE_TABLE notification // and persistLastProcessedNotificationID is explicitly invoked verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(), - Mockito.any(Updateable.Update.class)); + Mockito.any(UniquePathsUpdate.class)); //noinspection unchecked verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); reset(sentryStore); // Create a table @@ -480,6 +480,6 @@ public class TestNotificationProcessor { // and persistLastProcessedNotificationID was not invoked. //noinspection unchecked verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + Mockito.anyCollection(), Mockito.any(UniquePathsUpdate.class)); } }