From commits-return-6850-archive-asf-public=cust-asf.ponee.io@kudu.apache.org Fri Jan 11 00:05:21 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 40E051807A6 for ; Fri, 11 Jan 2019 00:05:19 +0100 (CET) Received: (qmail 31727 invoked by uid 500); 10 Jan 2019 23:05:18 -0000 Mailing-List: contact commits-help@kudu.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kudu.apache.org Delivered-To: mailing list commits@kudu.apache.org Received: (qmail 31616 invoked by uid 99); 10 Jan 2019 23:05:18 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Jan 2019 23:05:18 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 347AE8704A; Thu, 10 Jan 2019 23:05:17 +0000 (UTC) Date: Thu, 10 Jan 2019 23:05:24 +0000 To: "commits@kudu.apache.org" Subject: [kudu] 08/09: [backup] Make the scanRequestTimeout unit clear MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: granthenke@apache.org In-Reply-To: <154716151695.7488.510925107260272458@gitbox.apache.org> References: <154716151695.7488.510925107260272458@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: kudu X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Rev: 61c44785763219df1225108dbde6ac6ada70e087 X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20190110230517.347AE8704A@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. granthenke pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git commit 61c44785763219df1225108dbde6ac6ada70e087 Author: Grant Henke AuthorDate: Wed Jan 9 15:38:57 2019 -0600 [backup] Make the scanRequestTimeout unit clear Changes the scanRequestTimeout parameter to be scanRequestTimeoutMs to make sure the time unit is clear. Change-Id: Id44832a06ae5fb4762a402faff82012729afb5c4 Reviewed-on: http://gerrit.cloudera.org:8080/12210 Tested-by: Kudu Jenkins Reviewed-by: Adar Dembo --- .../src/main/scala/org/apache/kudu/backup/KuduBackup.scala | 7 +++++-- .../main/scala/org/apache/kudu/backup/KuduBackupOptions.scala | 10 +++++----- .../src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala index 55541a2..5a30f15 100644 --- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala +++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala @@ -40,11 +40,14 @@ object KuduBackup { def run(options: KuduBackupOptions, session: SparkSession): Unit = { val context = - new KuduContext(options.kuduMasterAddresses, session.sparkContext, + new KuduContext( + options.kuduMasterAddresses, + session.sparkContext, // TODO: As a workaround for KUDU-1868 the socketReadTimeout is // matched to the scanRequestTimeout. Without this // "Invalid call sequence ID" errors can occur under heavy load. - Some(options.scanRequestTimeout)) + Some(options.scanRequestTimeoutMs) + ) val path = options.path log.info(s"Backing up to path: $path") diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala index 82a8eb4..d366c7c 100644 --- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala +++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupOptions.scala @@ -32,14 +32,14 @@ case class KuduBackupOptions( timestampMs: Long = System.currentTimeMillis(), format: String = KuduBackupOptions.DefaultFormat, scanBatchSize: Int = KuduBackupOptions.DefaultScanBatchSize, - scanRequestTimeout: Long = KuduBackupOptions.DefaultScanRequestTimeout, + scanRequestTimeoutMs: Long = KuduBackupOptions.DefaultScanRequestTimeoutMs, scanPrefetching: Boolean = KuduBackupOptions.DefaultScanPrefetching, keepAlivePeriodMs: Long = KuduBackupOptions.defaultKeepAlivePeriodMs) object KuduBackupOptions { val DefaultFormat: String = "parquet" val DefaultScanBatchSize: Int = 1024 * 1024 * 20 // 20 MiB - val DefaultScanRequestTimeout: Long = + val DefaultScanRequestTimeoutMs: Long = AsyncKuduClient.DEFAULT_OPERATION_TIMEOUT_MS // 30 seconds val DefaultScanPrefetching : Boolean = false // TODO: Add a test per KUDU-1260 and enable by default? @@ -75,9 +75,9 @@ object KuduBackupOptions { .text("The maximum number of bytes returned by the scanner, on each batch.") .optional() - opt[Int]("scanRequestTimeout") - .action((v, o) => o.copy(scanRequestTimeout = v)) - .text("Sets how long each scan request to a server can last.") + opt[Int]("scanRequestTimeoutMs") + .action((v, o) => o.copy(scanRequestTimeoutMs = v)) + .text("Sets how long in milliseconds each scan request to a server can last.") .optional() opt[Unit]("scanPrefetching") diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala index 9be2bf2..2e45827 100644 --- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala +++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackupRDD.scala @@ -65,7 +65,7 @@ class KuduBackupRDD private[kudu] ( .readMode(ReadMode.READ_AT_SNAPSHOT) .snapshotTimestampRaw(hybridTime) .batchSizeBytes(options.scanBatchSize) - .scanRequestTimeout(options.scanRequestTimeout) + .scanRequestTimeout(options.scanRequestTimeoutMs) .prefetching(options.scanPrefetching) .build()