spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From andrewo...@apache.org
Subject git commit: [HOT FIX] Yarn stable tests don't compile
Date Fri, 31 Oct 2014 21:36:59 GMT
Repository: spark
Updated Branches:
  refs/heads/master 55ab77707 -> 087e31a7a


[HOT FIX] Yarn stable tests don't compile

This is caused by this commit: acd4ac7c9a503445e27739708cf36e19119b8ddc

Author: andrewor14 <andrew@databricks.com>
Author: Andrew Or <andrew@databricks.com>

Closes #3041 from andrewor14/yarn-hot-fix and squashes the following commits:

e5deba1 [andrewor14] Add new line at the end (minor)
aa998e8 [Andrew Or] Compilation hot fix


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/087e31a7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/087e31a7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/087e31a7

Branch: refs/heads/master
Commit: 087e31a7a0550f520c2e21b0ceaf36b969e050b4
Parents: 55ab777
Author: andrewor14 <andrew@databricks.com>
Authored: Fri Oct 31 14:36:55 2014 -0700
Committer: Andrew Or <andrew@databricks.com>
Committed: Fri Oct 31 14:36:55 2014 -0700

----------------------------------------------------------------------
 .../spark/deploy/yarn/YarnAllocator.scala       | 30 +++++++++++---------
 .../spark/deploy/yarn/YarnAllocatorSuite.scala  |  4 +--
 2 files changed, 19 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/087e31a7/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index e619619..b32e157 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -64,6 +64,8 @@ private[yarn] abstract class YarnAllocator(
     securityMgr: SecurityManager)
   extends Logging {
 
+  import YarnAllocator._
+
   // These three are locked on allocatedHostToContainersMap. Complementary data structures
   // allocatedHostToContainersMap : containers which are running : host, Set<containerid>
   // allocatedContainerToHostMap: container to host mapping.
@@ -439,19 +441,6 @@ private[yarn] abstract class YarnAllocator(
     }
   }
 
-  private val MEM_REGEX = "[0-9.]+ [KMG]B"
-  private val PMEM_EXCEEDED_PATTERN =
-    Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used")
-  private val VMEM_EXCEEDED_PATTERN =
-    Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used")
-
-  def memLimitExceededLogMessage(diagnostics: String, pattern: Pattern): String = {
-    val matcher = pattern.matcher(diagnostics)
-    val diag = if (matcher.find()) " " + matcher.group() + "." else ""
-    ("Container killed by YARN for exceeding memory limits." + diag
-      + " Consider boosting spark.yarn.executor.memoryOverhead.")
-  }
-
   protected def allocatedContainersOnHost(host: String): Int = {
     var retval = 0
     allocatedHostToContainersMap.synchronized {
@@ -532,3 +521,18 @@ private[yarn] abstract class YarnAllocator(
   }
 
 }
+
+private object YarnAllocator {
+  val MEM_REGEX = "[0-9.]+ [KMG]B"
+  val PMEM_EXCEEDED_PATTERN =
+    Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used")
+  val VMEM_EXCEEDED_PATTERN =
+    Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used")
+
+  def memLimitExceededLogMessage(diagnostics: String, pattern: Pattern): String = {
+    val matcher = pattern.matcher(diagnostics)
+    val diag = if (matcher.find()) " " + matcher.group() + "." else ""
+    ("Container killed by YARN for exceeding memory limits." + diag
+      + " Consider boosting spark.yarn.executor.memoryOverhead.")
+  }
+}

http://git-wip-us.apache.org/repos/asf/spark/blob/087e31a7/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
index 9fff63f..8d184a0 100644
--- a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
+++ b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.deploy.yarn
 
-import org.apache.spark.deploy.yarn.MemLimitLogger._
+import org.apache.spark.deploy.yarn.YarnAllocator._
 import org.scalatest.FunSuite
 
 class YarnAllocatorSuite extends FunSuite {
@@ -31,4 +31,4 @@ class YarnAllocatorSuite extends FunSuite {
     assert(vmemMsg.contains("5.8 GB of 4.2 GB virtual memory used."))
     assert(pmemMsg.contains("2.1 MB of 2 GB physical memory used."))
   }
-}
\ No newline at end of file
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message