Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E8B81200D43 for ; Tue, 21 Nov 2017 22:08:34 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id E7177160BFC; Tue, 21 Nov 2017 21:08:34 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3A939160BE3 for ; Tue, 21 Nov 2017 22:08:34 +0100 (CET) Received: (qmail 64249 invoked by uid 500); 21 Nov 2017 21:08:33 -0000 Mailing-List: contact commits-help@kudu.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kudu.apache.org Delivered-To: mailing list commits@kudu.apache.org Received: (qmail 64240 invoked by uid 99); 21 Nov 2017 21:08:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Nov 2017 21:08:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BCA5ADFBCA; Tue, 21 Nov 2017 21:08:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: danburkert@apache.org To: commits@kudu.apache.org Message-Id: <0e9a051c41fc429bab79539c40b67f81@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: kudu git commit: [spark] Remove syncCache in KuduConnection Date: Tue, 21 Nov 2017 21:08:31 +0000 (UTC) archived-at: Tue, 21 Nov 2017 21:08:35 -0000 Repository: kudu Updated Branches: refs/heads/master f0d52ab64 -> 48eab213c [spark] Remove syncCache in KuduConnection This patch removes syncCache in KuduConnection and uses the same AsyncKyduClient instance for both syncClient and asyncClient in KuduContext to avoid misuse. Change-Id: Ica8745d8503a35e17a632d0a0cde5738915f00fb Reviewed-on: http://gerrit.cloudera.org:8080/8551 Reviewed-by: Dan Burkert Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/48eab213 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/48eab213 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/48eab213 Branch: refs/heads/master Commit: 48eab213ca9014f445e41ee7d9d8b5b43aff8f42 Parents: f0d52ab Author: hahao Authored: Tue Nov 21 11:52:58 2017 -0800 Committer: Dan Burkert Committed: Tue Nov 21 21:08:18 2017 +0000 ---------------------------------------------------------------------- .../apache/kudu/spark/kudu/KuduContext.scala | 22 +------------------- .../kudu/spark/kudu/KuduContextTest.scala | 1 - 2 files changed, 1 insertion(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/48eab213/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala index 71eb6e8..6961430 100644 --- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala +++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala @@ -55,13 +55,7 @@ class KuduContext(val kuduMaster: String, this(kuduMaster, new SparkContext()) } - @transient lazy val syncClient: KuduClient = { - val c = KuduConnection.getSyncClient(kuduMaster) - if (authnCredentials != null) { - c.importAuthenticationCredentials(authnCredentials) - } - c - } + @transient lazy val syncClient: KuduClient = asyncClient.syncClient() @transient lazy val asyncClient: AsyncKuduClient = { val c = KuduConnection.getAsyncClient(kuduMaster) @@ -308,7 +302,6 @@ private object KuduContext { } private object KuduConnection { - private[kudu] val syncCache = new mutable.HashMap[String, KuduClient]() private[kudu] val asyncCache = new mutable.HashMap[String, AsyncKuduClient]() /** @@ -320,19 +313,6 @@ private object KuduConnection { */ private val ShutdownHookPriority = 100 - def getSyncClient(kuduMaster: String): KuduClient = { - syncCache.synchronized { - if (!syncCache.contains(kuduMaster)) { - val syncClient = new KuduClient.KuduClientBuilder(kuduMaster).build() - ShutdownHookManager.get().addShutdownHook(new Runnable { - override def run(): Unit = syncClient.close() - }, ShutdownHookPriority) - syncCache.put(kuduMaster, syncClient) - } - return syncCache(kuduMaster) - } - } - def getAsyncClient(kuduMaster: String): AsyncKuduClient = { asyncCache.synchronized { if (!asyncCache.contains(kuduMaster)) { http://git-wip-us.apache.org/repos/asf/kudu/blob/48eab213/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala index 620cf3f..8156365 100644 --- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala +++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/KuduContextTest.scala @@ -50,7 +50,6 @@ class KuduContextTest extends FunSuite with TestContext with Matchers { test("Test KuduContext serialization") { val serialized = serialize(kuduContext) - KuduConnection.syncCache.clear() KuduConnection.asyncCache.clear() val deserialized = deserialize(serialized).asInstanceOf[KuduContext] assert(deserialized.authnCredentials != null)