pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1720722 - /pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Date Fri, 18 Dec 2015 05:31:20 GMT
Author: xuefu
Date: Fri Dec 18 05:31:19 2015
New Revision: 1720722

URL: http://svn.apache.org/viewvc?rev=1720722&view=rev
Log:
PIG-4754: Fix UT failures in TestScriptLanguage (Xianda via Xuefu)

Modified:
    pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java

Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1720722&r1=1720721&r2=1720722&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
(original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Fri Dec 18 05:31:19 2015
@@ -371,17 +371,20 @@ public class SparkLauncher extends Launc
                     && jarFile.exists()) {
                 return;
             }
-            if (localFile.exists()) {
-                LOG.info(String.format(
-                        "jar file %s exists, ready to delete",
-                        localFile.getAbsolutePath()));
-                localFile.delete();
-            } else {
-                LOG.info(String.format("jar file %s not exists,",
-                        localFile.getAbsolutePath()));
+            // When multiple threads start SparkLauncher, delete/copy actions should be in
a critical section
+            synchronized(SparkLauncher.class) {
+                if (localFile.exists()) {
+                    LOG.info(String.format(
+                            "jar file %s exists, ready to delete",
+                            localFile.getAbsolutePath()));
+                    localFile.delete();
+                } else {
+                    LOG.info(String.format("jar file %s not exists,",
+                            localFile.getAbsolutePath()));
+                }
+                Files.copy(Paths.get(new Path(jarFile.getAbsolutePath()).toString()),
+                        Paths.get(localFile.getAbsolutePath()));
             }
-            Files.copy(Paths.get(new Path(jarFile.getAbsolutePath()).toString()),
-                    Paths.get(localFile.getAbsolutePath()));
         } else {
             sparkContext.addFile(jarFile.toURI().toURL()
                 .toExternalForm());
@@ -431,7 +434,11 @@ public class SparkLauncher extends Launc
         return sparkPlan;
     }
 
-    private static void startSparkIfNeeded(PigContext pc) throws PigException {
+    /**
+     * Only one SparkContext may be active per JVM (SPARK-2243). When multiple threads start
SparkLaucher,
+     * the static member sparkContext should be initialized only once
+     */
+    private static synchronized void startSparkIfNeeded(PigContext pc) throws PigException
{
         if (sparkContext == null) {
             String master = null;
             if (pc.getExecType().isLocal()) {



Mime
View raw message