Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 85146200C5B for ; Wed, 5 Apr 2017 00:41:44 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 8388B160BA1; Tue, 4 Apr 2017 22:41:44 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7D13F160B90 for ; Wed, 5 Apr 2017 00:41:43 +0200 (CEST) Received: (qmail 84221 invoked by uid 500); 4 Apr 2017 22:41:42 -0000 Mailing-List: contact commits-help@sentry.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@sentry.apache.org Delivered-To: mailing list commits@sentry.apache.org Received: (qmail 84212 invoked by uid 99); 4 Apr 2017 22:41:42 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Apr 2017 22:41:42 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9591EDFE34; Tue, 4 Apr 2017 22:41:42 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: akolb@apache.org To: commits@sentry.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: sentry git commit: SENTRY-1677: Add metrics to measure how much time to get Delta Path/Perm Updates (Alex Kolabsov, reviewed by Na Li) Date: Tue, 4 Apr 2017 22:41:42 +0000 (UTC) archived-at: Tue, 04 Apr 2017 22:41:44 -0000 Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 15613a1cc -> 9739dd80a SENTRY-1677: Add metrics to measure how much time to get Delta Path/Perm Updates (Alex Kolabsov, reviewed by Na Li) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/9739dd80 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/9739dd80 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/9739dd80 Branch: refs/heads/sentry-ha-redesign Commit: 9739dd80a1635dc07f71f05857b5d3278872d1ca Parents: 15613a1 Author: Alexander Kolbasov Authored: Tue Apr 4 15:40:46 2017 -0700 Committer: Alexander Kolbasov Committed: Tue Apr 4 15:40:46 2017 -0700 ---------------------------------------------------------------------- .../apache/sentry/hdfs/PathDeltaRetriever.java | 45 +++++++++++--------- .../apache/sentry/hdfs/PermDeltaRetriever.java | 41 ++++++++++-------- .../sentry/hdfs/SentryHdfsMetricsUtil.java | 37 ++++++++++++---- 3 files changed, 78 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java index cea5b9d..5425114 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PathDeltaRetriever.java @@ -17,6 +17,7 @@ */ package org.apache.sentry.hdfs; +import com.codahale.metrics.Timer; import org.apache.sentry.provider.db.service.model.MSentryPathChange; import org.apache.sentry.provider.db.service.persistent.SentryStore; @@ -43,34 +44,40 @@ public class PathDeltaRetriever implements DeltaRetriever { @Override public Collection retrieveDelta(long seqNum) throws Exception { - Collection mSentryPathChanges = - sentryStore.getMSentryPathChanges(seqNum); - if (mSentryPathChanges.isEmpty()) { - return Collections.emptyList(); - } + try (final Timer.Context timerContext = + SentryHdfsMetricsUtil.getDeltaPathChangesTimer.time()) { + Collection mSentryPathChanges = + sentryStore.getMSentryPathChanges(seqNum); + + SentryHdfsMetricsUtil.getDeltaPathChangesHistogram.update(mSentryPathChanges.size()); + + if (mSentryPathChanges.isEmpty()) { + return Collections.emptyList(); + } - Collection updates = new ArrayList<>(mSentryPathChanges.size()); - for (MSentryPathChange mSentryPathChange : mSentryPathChanges) { - // Gets the changeID from the persisted MSentryPathChange. - long changeID = mSentryPathChange.getChangeID(); - // Creates a corresponding PathsUpdate and deserialize the - // persisted delta update in JSON format to TPathsUpdate with - // associated changeID. - PathsUpdate pathsUpdate = new PathsUpdate(); - pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange()); - pathsUpdate.setSeqNum(changeID); - updates.add(pathsUpdate); + Collection updates = new ArrayList<>(mSentryPathChanges.size()); + for (MSentryPathChange mSentryPathChange : mSentryPathChanges) { + // Gets the changeID from the persisted MSentryPathChange. + long changeID = mSentryPathChange.getChangeID(); + // Creates a corresponding PathsUpdate and deserialize the + // persisted delta update in JSON format to TPathsUpdate with + // associated changeID. + PathsUpdate pathsUpdate = new PathsUpdate(); + pathsUpdate.JSONDeserialize(mSentryPathChange.getPathChange()); + pathsUpdate.setSeqNum(changeID); + updates.add(pathsUpdate); + } + return updates; } - return updates; } @Override - public boolean isDeltaAvailable(long seqNum) throws Exception { + public boolean isDeltaAvailable ( long seqNum) throws Exception { return sentryStore.pathChangeExists(seqNum); } @Override - public long getLatestDeltaID() throws Exception { + public long getLatestDeltaID () throws Exception { return sentryStore.getLastProcessedPathChangeID(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java index 9649b02..a29fc74 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PermDeltaRetriever.java @@ -17,6 +17,7 @@ */ package org.apache.sentry.hdfs; +import com.codahale.metrics.Timer; import org.apache.sentry.provider.db.service.model.MSentryPermChange; import org.apache.sentry.provider.db.service.persistent.SentryStore; @@ -43,25 +44,31 @@ public class PermDeltaRetriever implements DeltaRetriever { @Override public Collection retrieveDelta(long seqNum) throws Exception { - Collection mSentryPermChanges = - sentryStore.getMSentryPermChanges(seqNum); - if (mSentryPermChanges.isEmpty()) { - return Collections.emptyList(); - } + try (final Timer.Context timerContext = + SentryHdfsMetricsUtil.getDeltaPermChangesTimer.time()) { + Collection mSentryPermChanges = + sentryStore.getMSentryPermChanges(seqNum); + + SentryHdfsMetricsUtil.getDeltaPermChangesHistogram.update(mSentryPermChanges.size()); + + if (mSentryPermChanges.isEmpty()) { + return Collections.emptyList(); + } - Collection updates = new ArrayList<>(mSentryPermChanges.size()); - for (MSentryPermChange mSentryPermChange : mSentryPermChanges) { - // Get the changeID from the persisted MSentryPermChange - long changeID = mSentryPermChange.getChangeID(); - // Create a corresponding PermissionsUpdate and deserialize the - // persisted delta update in JSON format to TPermissionsUpdate with - // associated changeID. - PermissionsUpdate permsUpdate = new PermissionsUpdate(); - permsUpdate.JSONDeserialize(mSentryPermChange.getPermChange()); - permsUpdate.setSeqNum(changeID); - updates.add(permsUpdate); + Collection updates = new ArrayList<>(mSentryPermChanges.size()); + for (MSentryPermChange mSentryPermChange : mSentryPermChanges) { + // Get the changeID from the persisted MSentryPermChange + long changeID = mSentryPermChange.getChangeID(); + // Create a corresponding PermissionsUpdate and deserialize the + // persisted delta update in JSON format to TPermissionsUpdate with + // associated changeID. + PermissionsUpdate permsUpdate = new PermissionsUpdate(); + permsUpdate.JSONDeserialize(mSentryPermChange.getPermChange()); + permsUpdate.setSeqNum(changeID); + updates.add(permsUpdate); + } + return updates; } - return updates; } @Override http://git-wip-us.apache.org/repos/asf/sentry/blob/9739dd80/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java index 28bf20e..932a5c0 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java @@ -27,20 +27,20 @@ import org.apache.sentry.provider.db.service.thrift.SentryMetrics; /** * Util class to support metrics. */ -public class SentryHdfsMetricsUtil { +final class SentryHdfsMetricsUtil { // SentryMetrics private static final SentryMetrics sentryMetrics = SentryMetrics.getInstance(); // Metrics for get_all_authz_updates_from in SentryHDFSServiceProcessor // The time used for each get_all_authz_updates_from - public static final Timer getAllAuthzUpdatesTimer = sentryMetrics.getTimer( + static final Timer getAllAuthzUpdatesTimer = sentryMetrics.getTimer( MetricRegistry.name(SentryHDFSServiceProcessor.class, "get-all-authz-updates-from")); // The size of perm updates for each get_all_authz_updates_from - public static final Histogram getPermUpdateHistogram = sentryMetrics.getHistogram( + static final Histogram getPermUpdateHistogram = sentryMetrics.getHistogram( MetricRegistry.name(SentryHDFSServiceProcessor.class, "perm-updates-size")); // The size of path updates for each get_all_authz_updates_from - public static final Histogram getPathUpdateHistogram = sentryMetrics.getHistogram( + static final Histogram getPathUpdateHistogram = sentryMetrics.getHistogram( MetricRegistry.name(SentryHDFSServiceProcessor.class, "paths-updates-size")); // Metrics for handle_hms_notification in SentryHDFSServiceProcessor @@ -62,27 +62,46 @@ public class SentryHdfsMetricsUtil { // Metrics for retrievePermFullImage in PermImageRetriever // The time used for each retrievePermFullImage - public static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer( + static final Timer getRetrievePermFullImageTimer = sentryMetrics.getTimer( MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image")); // The size of privilege changes for each retrievePermFullImage - public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram( + static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram( MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "privilege-changes-size")); // The size of role changes for each retrievePermFullImage call - public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram( + static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram( MetricRegistry.name(PermImageRetriever.class, "retrieve-perm-full-image", "role-changes-size")); // Metrics for retrievePathFullImage in PathImageRetriever // The time used for each retrievePathFullImage - public static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer( + static final Timer getRetrievePathFullImageTimer = sentryMetrics.getTimer( MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image")); // The size of path changes for each retrievePathFullImage - public static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram( + static final Histogram getPathChangesHistogram = sentryMetrics.getHistogram( MetricRegistry.name(PathImageRetriever.class, "retrieve-path-full-image", "path-changes-size")); + // Timer for getting path changes deltas + static final Timer getDeltaPathChangesTimer = sentryMetrics.getTimer( + MetricRegistry.name(PathDeltaRetriever.class, "path", "delta", "time") + ); + + // Histogram for the number of path changes processed for deltas + static final Histogram getDeltaPathChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(PathDeltaRetriever.class, "path", "delta", "size")); + + + // Timer for getting permission changes deltas + static final Timer getDeltaPermChangesTimer = sentryMetrics.getTimer( + MetricRegistry.name(PathDeltaRetriever.class, "perm", "delta", "time") + ); + + // Histogram for the number of permissions changes processed for deltas + static final Histogram getDeltaPermChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(PathDeltaRetriever.class, "perm", "delta", "size")); + private SentryHdfsMetricsUtil() { // Make constructor private to avoid instantiation }