From issues-return-18670-archive-asf-public=cust-asf.ponee.io@hawq.incubator.apache.org Thu Jul 12 17:12:28 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 38204180654 for ; Thu, 12 Jul 2018 17:12:28 +0200 (CEST) Received: (qmail 46017 invoked by uid 500); 12 Jul 2018 15:12:27 -0000 Mailing-List: contact issues-help@hawq.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hawq.incubator.apache.org Delivered-To: mailing list issues@hawq.incubator.apache.org Received: (qmail 46008 invoked by uid 99); 12 Jul 2018 15:12:27 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Jul 2018 15:12:27 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 3D3F5180028 for ; Thu, 12 Jul 2018 15:12:26 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4 X-Spam-Level: X-Spam-Status: No, score=-4 tagged_above=-999 required=6.31 tests=[KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id f0EmszJkvf6g for ; Thu, 12 Jul 2018 15:12:24 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 144BD5F1F0 for ; Thu, 12 Jul 2018 15:12:23 +0000 (UTC) Received: (qmail 45716 invoked by uid 99); 12 Jul 2018 15:12:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Jul 2018 15:12:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E21ABDFCBA; Thu, 12 Jul 2018 15:12:22 +0000 (UTC) From: shivzone To: issues@hawq.incubator.apache.org Reply-To: issues@hawq.incubator.apache.org References: In-Reply-To: Subject: [GitHub] incubator-hawq pull request #1379: HAWQ-1622. Cache PXF proxy UGI so that cl... Content-Type: text/plain Message-Id: <20180712151222.E21ABDFCBA@git1-us-west.apache.org> Date: Thu, 12 Jul 2018 15:12:22 +0000 (UTC) Github user shivzone commented on a diff in the pull request: https://github.com/apache/incubator-hawq/pull/1379#discussion_r202065515 --- Diff: pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/UGICacheTest.java --- @@ -0,0 +1,264 @@ +package org.apache.hawq.pxf.service; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.google.common.base.Ticker; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; + +public class UGICacheTest { + private static final long MINUTES = 60 * 1000L; + private UGIProvider provider = null; + private SessionId session = null; + private UGICache cache = null; + private FakeTicker fakeTicker; + + private static class FakeTicker extends Ticker { + private final AtomicLong nanos = new AtomicLong(); + + @Override + public long read() { + return nanos.get(); + } + + long advanceTime(long milliseconds) { + return nanos.addAndGet(milliseconds * 1000) / 1000; + } + } + + @Before + public void setUp() throws Exception { + provider = mock(UGIProvider.class); + when(provider.createProxyUGI(any(String.class))).thenAnswer(new Answer() { + @Override + public UserGroupInformation answer(InvocationOnMock invocation) { + return mock(UserGroupInformation.class); + } + }); + + session = new SessionId(0, "txn-id", "the-user"); + fakeTicker = new FakeTicker(); + cache = new UGICache(provider, fakeTicker); + } + + @Test + public void getUGIFromEmptyCache() throws Exception { + UserGroupInformation ugi = cache.getUserGroupInformation(session); + assertNotNull(ugi); + verify(provider).createProxyUGI("the-user"); + } + + @Test + public void getSameUGITwiceUsesCache() throws Exception { + UserGroupInformation ugi1 = cache.getUserGroupInformation(session); + UserGroupInformation ugi2 = cache.getUserGroupInformation(session); + assertEquals(ugi1, ugi2); + verify(provider, times(1)).createProxyUGI("the-user"); + } + + @Test + public void getTwoUGIsWithDifferentSessionsForSameUser() throws Exception { + SessionId otherSession = new SessionId(0, "txn-id-2", "the-user"); + UserGroupInformation proxyUGI1 = cache.getUserGroupInformation(session); + UserGroupInformation proxyUGI2 = cache.getUserGroupInformation(otherSession); + assertNotEquals(proxyUGI1, proxyUGI2); + verify(provider, times(2)).createProxyUGI("the-user"); + // TODO: this seems weird. We're creating two UGIs with the same params, + // even though we have two different sessions. Why? + } + + @Test + public void getTwoUGIsWithDifferentUsers() throws Exception { + SessionId otherSession = new SessionId(0, "txn-id", "different-user"); + UserGroupInformation proxyUGI1 = cache.getUserGroupInformation(session); + UserGroupInformation proxyUGI2 = cache.getUserGroupInformation(otherSession); + assertNotEquals(proxyUGI1, proxyUGI2); + verify(provider, times(1)).createProxyUGI("the-user"); + verify(provider, times(1)).createProxyUGI("different-user"); + } + + @Test + public void gettingTwoUGIsWithDifferentUsersCachesBoth() throws Exception { + SessionId otherSession = new SessionId(0, "txn-id", "different-user"); + UserGroupInformation ugi1RefA = cache.getUserGroupInformation(session); + UserGroupInformation ugi1RefB = cache.getUserGroupInformation(session); + UserGroupInformation ugi2RefA = cache.getUserGroupInformation(otherSession); + UserGroupInformation ugi2RefB = cache.getUserGroupInformation(otherSession); + assertSame(ugi1RefA, ugi1RefB); + assertSame(ugi2RefA, ugi2RefB); + assertNotEquals(ugi1RefA, ugi2RefA); + verify(provider, times(1)).createProxyUGI("the-user"); + verify(provider, times(1)).createProxyUGI("different-user"); + } + + @Test + public void anySegmentIdIsValid() throws Exception { + int crazySegId = Integer.MAX_VALUE; + session = new SessionId(crazySegId, "txn-id", "the-user"); + UserGroupInformation proxyUGI1 = cache.getUserGroupInformation(session); + assertNotNull(proxyUGI1); + } + + @Test + public void ensureCleanUpAfterExpiration() throws Exception { + UserGroupInformation ugi1 = cache.getUserGroupInformation(session); + + fakeTicker.advanceTime(10 * MINUTES); + cache.release(session, false); + fakeTicker.advanceTime(16 * MINUTES); + assertNotInCache(session, ugi1); + } + + @Test + public void releaseWithoutForceClean() throws Exception { + UserGroupInformation ugi1 = cache.getUserGroupInformation(session); + + cache.release(session, false); + // UGI wasn't cleaned up, so we can still get it + assertStillInCache(session, ugi1); + verify(provider, times(1)).createProxyUGI("the-user"); + } + + @Test + public void releaseWithForceClean() throws Exception { + UserGroupInformation ugi1 = cache.getUserGroupInformation(session); + + cache.release(session, true); + assertNotInCache(session, ugi1); + } + + @Test + public void releaseResetsTheExpirationTime() throws Exception { + UserGroupInformation reference1 = cache.getUserGroupInformation(session); + cache.getUserGroupInformation(session); + + cache.release(session, true); + fakeTicker.advanceTime(15 * MINUTES); --- End diff -- Can we instead use UGI_CACHE_EXPIRY here ---