falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-2048 Cluster submission failed in yarn-cluster mode
Date Fri, 01 Jul 2016 17:31:22 GMT
Repository: falcon
Updated Branches:
  refs/heads/master c12c999b4 -> 57fa2ff12


FALCON-2048 Cluster submission failed in yarn-cluster mode

Author: peeyush b <pbishnoi@hortonworks.com>

Reviewers: "Venkat Ranganathan <venkat@hortonworks.com>"

Closes #210 from peeyushb/FALCON-2048


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/57fa2ff1
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/57fa2ff1
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/57fa2ff1

Branch: refs/heads/master
Commit: 57fa2ff125602706b26b6d168a84b828c490ff74
Parents: c12c999
Author: peeyush b <pbishnoi@hortonworks.com>
Authored: Fri Jul 1 10:31:15 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Fri Jul 1 10:31:15 2016 -0700

----------------------------------------------------------------------
 .../falcon/entity/parser/ClusterEntityParser.java  | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/57fa2ff1/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
index 96ba748..3f35962 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/ClusterEntityParser.java
@@ -45,8 +45,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AuthorizationException;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaSparkContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -236,14 +234,13 @@ public class ClusterEntityParser extends EntityParser<Cluster>
{
     }
 
     protected void validateSparkMasterInterface(Cluster cluster) throws ValidationException
{
-        final String sparkMasterUrl = ClusterHelper.getSparkMasterEndPoint(cluster);
-        if (StringUtils.isNotEmpty(sparkMasterUrl)) {
-            SparkConf sparkConf = new SparkConf();
-            sparkConf.setMaster(sparkMasterUrl).setAppName("Falcon Spark");
-
-            JavaSparkContext sc = new JavaSparkContext(sparkConf);
-            if (sc.startTime() == null) {
-                throw new ValidationException("Unable to reach Spark master URL:" + sparkMasterUrl);
+        final String sparkMasterEndPoint = ClusterHelper.getSparkMasterEndPoint(cluster);
+        LOG.info("Validating spark interface: {}", sparkMasterEndPoint);
+        if (StringUtils.isNotEmpty(sparkMasterEndPoint)) {
+            if (!("yarn-cluster".equalsIgnoreCase(sparkMasterEndPoint)
+                    || "yarn-client".equalsIgnoreCase(sparkMasterEndPoint)
+                    || "local".equalsIgnoreCase(sparkMasterEndPoint))) {
+                throw new ValidationException("Invalid Spark Interface End Point:" + sparkMasterEndPoint);
             }
         }
     }


Mime
View raw message