spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From van...@apache.org
Subject spark git commit: [SPARK-22341][YARN] Impersonate correct user when preparing resources.
Date Wed, 25 Oct 2017 20:53:07 GMT
Repository: spark
Updated Branches:
  refs/heads/master 6ea8a56ca -> d212ef14b


[SPARK-22341][YARN] Impersonate correct user when preparing resources.

The bug was introduced in SPARK-22290, which changed how the app's user
is impersonated in the AM. The changed missed an initialization function
that needs to be run as the app owner (who has the right credentials to
read from HDFS).

Author: Marcelo Vanzin <vanzin@cloudera.com>

Closes #19566 from vanzin/SPARK-22341.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d212ef14
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d212ef14
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d212ef14

Branch: refs/heads/master
Commit: d212ef14be7c2864cc529e48a02a47584e46f7a5
Parents: 6ea8a56
Author: Marcelo Vanzin <vanzin@cloudera.com>
Authored: Wed Oct 25 13:53:01 2017 -0700
Committer: Marcelo Vanzin <vanzin@cloudera.com>
Committed: Wed Oct 25 13:53:01 2017 -0700

----------------------------------------------------------------------
 .../spark/deploy/yarn/ApplicationMaster.scala     | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/d212ef14/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index f616723..244d912 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -97,9 +97,7 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments)
extends
     }
   }
 
-  private val client = ugi.doAs(new PrivilegedExceptionAction[YarnRMClient]() {
-    def run: YarnRMClient = new YarnRMClient()
-  })
+  private val client = doAsUser { new YarnRMClient() }
 
   // Default to twice the number of executors (twice the maximum number of executors if dynamic
   // allocation is enabled), with a minimum of 3.
@@ -178,7 +176,7 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments)
extends
   // Load the list of localized files set by the client. This is used when launching executors,
   // and is loaded here so that these configs don't pollute the Web UI's environment page
in
   // cluster mode.
-  private val localResources = {
+  private val localResources = doAsUser {
     logInfo("Preparing Local resources")
     val resources = HashMap[String, LocalResource]()
 
@@ -240,9 +238,9 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments)
extends
   }
 
   final def run(): Int = {
-    ugi.doAs(new PrivilegedExceptionAction[Unit]() {
-      def run: Unit = runImpl()
-    })
+    doAsUser {
+      runImpl()
+    }
     exitCode
   }
 
@@ -790,6 +788,12 @@ private[spark] class ApplicationMaster(args: ApplicationMasterArguments)
extends
     }
   }
 
+  private def doAsUser[T](fn: => T): T = {
+    ugi.doAs(new PrivilegedExceptionAction[T]() {
+      override def run: T = fn
+    })
+  }
+
 }
 
 object ApplicationMaster extends Logging {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message