From issues-return-18638-archive-asf-public=cust-asf.ponee.io@hawq.incubator.apache.org Wed Jul 11 01:24:03 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id B25671807A6 for ; Wed, 11 Jul 2018 01:24:01 +0200 (CEST) Received: (qmail 47884 invoked by uid 500); 10 Jul 2018 23:24:00 -0000 Mailing-List: contact issues-help@hawq.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hawq.incubator.apache.org Delivered-To: mailing list issues@hawq.incubator.apache.org Received: (qmail 47813 invoked by uid 99); 10 Jul 2018 23:24:00 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Jul 2018 23:24:00 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id 381ED1A2B18 for ; Tue, 10 Jul 2018 23:24:00 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4 X-Spam-Level: X-Spam-Status: No, score=-4 tagged_above=-999 required=6.31 tests=[KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id YSiCmYfi8fMr for ; Tue, 10 Jul 2018 23:23:59 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id 5B7495F42E for ; Tue, 10 Jul 2018 23:23:58 +0000 (UTC) Received: (qmail 47006 invoked by uid 99); 10 Jul 2018 23:23:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Jul 2018 23:23:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C0722DFAD1; Tue, 10 Jul 2018 23:23:57 +0000 (UTC) From: denalex To: issues@hawq.incubator.apache.org Reply-To: issues@hawq.incubator.apache.org References: In-Reply-To: Subject: [GitHub] incubator-hawq pull request #1379: HAWQ-1622. Cache PXF proxy UGI so that cl... Content-Type: text/plain Message-Id: <20180710232357.C0722DFAD1@git1-us-west.apache.org> Date: Tue, 10 Jul 2018 23:23:57 +0000 (UTC) Github user denalex commented on a diff in the pull request: https://github.com/apache/incubator-hawq/pull/1379#discussion_r201517046 --- Diff: pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/UGICache.java --- @@ -0,0 +1,143 @@ +package org.apache.hawq.pxf.service; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.DelayQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.security.UserGroupInformation; + +public class UGICache { + + private static final Log LOG = LogFactory.getLog(UGICache.class); + private Map cache = new ConcurrentHashMap<>(); + @SuppressWarnings("unchecked") + // There is a separate DelayQueue for each segment (also being used for locking) + private DelayQueue[] delayQueues = (DelayQueue[])new DelayQueue[64]; + private final UGIProvider ugiProvider; + + public UGICache(UGIProvider provider) { + this.ugiProvider = provider; + for (int i = 0; i < delayQueues.length; i++) { + delayQueues[i] = new DelayQueue<>(); + } + } + + public UGICache() { + this(new UGIProvider()); + } + + // Create new proxy UGI if not found in cache and increment reference count + public UGICacheEntry getTimedProxyUGI(SessionId session) + throws IOException { + + Integer segmentId = session.getSegmentId(); + String user = session.getUser(); + synchronized (delayQueues[segmentId]) { + // Use the opportunity to cleanup any expired entries + cleanup(segmentId); + UGICacheEntry timedProxyUGI = cache.get(session); + if (timedProxyUGI == null) { + LOG.info(session.toString() + " Creating proxy user = " + user); + UserGroupInformation proxyUGI = ugiProvider.createProxyUGI(user); + timedProxyUGI = new UGICacheEntry(proxyUGI, session); + delayQueues[segmentId].offer(timedProxyUGI); + cache.put(session, timedProxyUGI); + } + timedProxyUGI.incrementCounter(); + return timedProxyUGI; + } + } + + // Poll segment expiration queue for all expired entries + // and clean them if possible + private void cleanup(Integer segmentId) { --- End diff -- proper javadoc, please ---