kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [5/8] kylin git commit: KYLIN-2503 Spark cubing step should show YARN app link
Date Sun, 12 Mar 2017 02:51:09 GMT
KYLIN-2503 Spark cubing step should show YARN app link


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f38d871a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f38d871a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f38d871a

Branch: refs/heads/KYLIN-2497-2
Commit: f38d871a72e1e24331e092fd51664fcb8fd47e22
Parents: 5d012b9
Author: shaofengshi <shaofengshi@apache.org>
Authored: Sat Mar 11 22:08:28 2017 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Sat Mar 11 22:08:28 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/job/common/ShellExecutable.java   | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/f38d871a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
index a68f242..5e0d0db 100644
--- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java
@@ -83,6 +83,10 @@ public class ShellExecutable extends AbstractExecutable {
         private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting
Job = (.*?), Tracking URL = (.*)");
         private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS
Read: (\\d+) HDFS Write: (\\d+) SUCCESS");
 
+        // spark
+        private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application
(.*?)");
+        private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL:
(.*)");
+
         @Override
         public void log(String message) {
             Matcher matcher = PATTERN_APP_ID.matcher(message);
@@ -136,6 +140,19 @@ public class ShellExecutable extends AbstractExecutable {
                 String hdfsWritten = matcher.group(2);
                 info.put(ExecutableConstants.HDFS_BYTES_WRITTEN, hdfsWritten);
             }
+
+            // spark
+            matcher = PATTERN_SPARK_APP_ID.matcher(message);
+            if (matcher.find()) {
+                String app_id = matcher.group(1);
+                info.put(ExecutableConstants.YARN_APP_ID, app_id);
+            }
+
+            matcher = PATTERN_SPARK_APP_URL.matcher(message);
+            if (matcher.find()) {
+                String trackingUrl = matcher.group(1);
+                info.put(ExecutableConstants.YARN_APP_URL, trackingUrl);
+            }
         }
 
         Map<String, String> getInfo() {


Mime
View raw message