kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From danburk...@apache.org
Subject [2/2] kudu git commit: kudu-spark: change default master address to local fqdn
Date Wed, 20 Sep 2017 21:26:44 GMT
kudu-spark: change default master address to local fqdn

The previous default was 'localhost', which isn't resolvable in a
cluster context. It would also cause issues on Kerberized clusters due
to 'localhost' not matching the Kerberos principal name of the master
(see KUDU-2142).

Change-Id: I10ec7414c451f54b95d86663d743162688e304ba
Reviewed-on: http://gerrit.cloudera.org:8080/8072
Reviewed-by: Attila Bukor <abukor@cloudera.com>
Reviewed-by: Todd Lipcon <todd@apache.org>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5a0015e7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5a0015e7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5a0015e7

Branch: refs/heads/master
Commit: 5a0015e75c6bde8ddb6f113083c3e7d46e48c5de
Parents: 3ab3f45
Author: Dan Burkert <danburkert@apache.org>
Authored: Thu Sep 14 11:43:38 2017 -0700
Committer: Dan Burkert <danburkert@apache.org>
Committed: Wed Sep 20 21:25:57 2017 +0000

----------------------------------------------------------------------
 .../apache/kudu/spark/tools/ImportExportFiles.scala | 10 +++++++---
 .../spark/tools/IntegrationTestBigLinkedList.scala  | 16 ++++++++++------
 .../org/apache/kudu/spark/kudu/DefaultSource.scala  | 10 ++++++----
 3 files changed, 23 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/5a0015e7/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
index 025b06b..a09b52d 100644
--- a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
+++ b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
@@ -17,9 +17,11 @@
 
 package org.apache.kudu.spark.tools
 
+import java.net.InetAddress
+
 import org.apache.kudu.client.KuduClient
 import org.apache.kudu.spark.tools.ImportExportKudu.ArgsCls
-import org.apache.spark.sql.{SQLContext, SparkSession}
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.SparkConf
 import org.slf4j.{Logger, LoggerFactory}
 import org.apache.kudu.spark.kudu._
@@ -33,13 +35,15 @@ object ImportExportKudu {
     sys.exit(1)
   }
 
+  def defaultMasterAddrs: String = InetAddress.getLocalHost.getCanonicalHostName
+
   def usage: String =
     s"""
        | Usage: --operation=import/export --format=<data-format(csv,parquet,avro)>
--master-addrs=<master-addrs> --path=<path> --table-name=<table-name>
        |    where
        |      operation: import or export data from or to Kudu tables, default: import
        |      format: specify the format of data want to import/export, the following formats
are supported csv,parquet,avro default:csv
-       |      masterAddrs: comma separated addresses of Kudu master nodes, default: localhost
+       |      masterAddrs: comma separated addresses of Kudu master nodes, default: $defaultMasterAddrs
        |      path: path to input or output for import/export operation, default: file://
        |      tableName: table name to import/export, default: ""
        |      columns: columns name for select statement on export from kudu table, default:
*
@@ -49,7 +53,7 @@ object ImportExportKudu {
 
   case class ArgsCls(operation: String = "import",
                      format: String = "csv",
-                     masterAddrs: String = "localhost",
+                     masterAddrs: String = defaultMasterAddrs,
                      path: String = "file://",
                      tableName: String = "",
                      columns: String = "*",

http://git-wip-us.apache.org/repos/asf/kudu/blob/5a0015e7/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
index db03c24..7b1fc31 100644
--- a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
+++ b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
@@ -17,6 +17,8 @@
 
 package org.apache.kudu.spark.tools
 
+import java.net.InetAddress
+
 import com.google.common.annotations.VisibleForTesting
 import org.apache.kudu.client.SessionConfiguration.FlushMode
 import org.apache.kudu.client.{KuduClient, KuduSession, KuduTable}
@@ -81,6 +83,8 @@ object IntegrationTestBigLinkedList {
     else s"${n}ns"
   }
 
+  def defaultMasterAddrs: String = InetAddress.getLocalHost.getCanonicalHostName
+
   def main(args: Array[String]): Unit = {
     if (args.isEmpty) { fail(usage) }
 
@@ -94,7 +98,7 @@ object IntegrationTestBigLinkedList {
 }
 
 object Generator {
-  import IntegrationTestBigLinkedList.{LOG, fail, nanosToHuman, parseIntFlag}
+  import IntegrationTestBigLinkedList.{LOG, defaultMasterAddrs, fail, nanosToHuman, parseIntFlag}
 
   def usage: String =
     s"""
@@ -108,7 +112,7 @@ object Generator {
        |      hashPartitions: number of hash partitions to create for the new linked list
table, if it doesn't exist, default: 1
        |      rangePartitions: number of range partitions to create for the new linked list
table, if it doesn't exist, default: 1
        |      replicas: number of replicas to create for the new linked list table, if it
doesn't exist, default: 1
-       |      master-addrs: comma separated addresses of Kudu master nodes, default: localhost
+       |      master-addrs: comma separated addresses of Kudu master nodes, default: $defaultMasterAddrs
        |      table-name: the name of the linked list table, default: $DEFAULT_TABLE_NAME
      """.stripMargin
 
@@ -118,7 +122,7 @@ object Generator {
                   hashPartitions: Int = 1,
                   rangePartitions: Int = 1,
                   replicas: Int = 1,
-                  masterAddrs: String = "localhost",
+                  masterAddrs: String = defaultMasterAddrs,
                   tableName: String = DEFAULT_TABLE_NAME)
 
   object Args {
@@ -267,19 +271,19 @@ object Generator {
 }
 
 object Verifier {
-  import IntegrationTestBigLinkedList.{fail, parseLongFlag}
+  import IntegrationTestBigLinkedList.{defaultMasterAddrs, fail, parseLongFlag}
 
   def usage: String =
     s"""
        | Usage: verify --nodes=<nodes> --master-addrs=<master-addrs> --table-name=<table-name>
        |    where
        |      nodes: number of nodes expected to be in the linked list table
-       |      master-addrs: comma separated addresses of Kudu master nodes, default: localhost
+       |      master-addrs: comma separated addresses of Kudu master nodes, default: $defaultMasterAddrs
        |      table-name: the name of the linked list table, default: $DEFAULT_TABLE_NAME
      """.stripMargin
 
   case class Args(nodes: Option[Long] = None,
-                  masterAddrs: String = "localhost",
+                  masterAddrs: String = defaultMasterAddrs,
                   tableName: String = DEFAULT_TABLE_NAME)
 
   object Args {

http://git-wip-us.apache.org/repos/asf/kudu/blob/5a0015e7/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index 5c91a1f..fbc2d38 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -17,6 +17,7 @@
 
 package org.apache.kudu.spark.kudu
 
+import java.net.InetAddress
 import java.sql.Timestamp
 
 import scala.collection.JavaConverters._
@@ -49,6 +50,8 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
   val OPERATION = "kudu.operation"
   val FAULT_TOLERANT_SCANNER = "kudu.faultTolerantScan"
 
+  def defaultMasterAddrs: String = InetAddress.getLocalHost.getCanonicalHostName
+
   /**
     * Construct a BaseRelation using the provided context and parameters.
     *
@@ -62,13 +65,12 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
     val tableName = parameters.getOrElse(TABLE_KEY,
       throw new IllegalArgumentException(
         s"Kudu table name must be specified in create options using key '$TABLE_KEY'"))
-    val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+    val kuduMaster = parameters.getOrElse(KUDU_MASTER, defaultMasterAddrs)
     val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
     val faultTolerantScanner = Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
       .getOrElse(false)
 
-    new KuduRelation(tableName, kuduMaster, faultTolerantScanner, operationType,
-      None)(sqlContext)
+    new KuduRelation(tableName, kuduMaster, faultTolerantScanner, operationType, None)(sqlContext)
   }
 
   /**
@@ -97,7 +99,7 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
     val tableName = parameters.getOrElse(TABLE_KEY,
       throw new IllegalArgumentException(s"Kudu table name must be specified in create options
" +
         s"using key '$TABLE_KEY'"))
-    val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+    val kuduMaster = parameters.getOrElse(KUDU_MASTER, defaultMasterAddrs)
     val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
     val faultTolerantScanner = Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
       .getOrElse(false)


Mime
View raw message