eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jinh...@apache.org
Subject incubator-eagle git commit: [EAGLE-728] Application can't load application.properties when running local mode
Date Wed, 09 Nov 2016 02:47:13 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 6ec928941 -> ab3ed13e1


[EAGLE-728] Application can't load application.properties when running local mode

EAGLE-728 Application can't load application.properties when running local mode
- Use hard code, and overwrite it if user passes this configuration form web ui.

https://issues.apache.org/jira/browse/EAGLE-728

Author: chitin <chitin1027@gmail.com>

Closes #612 from chitin/EAGLE-728-2.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/ab3ed13e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/ab3ed13e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/ab3ed13e

Branch: refs/heads/master
Commit: ab3ed13e18c85fa3fd64a08a070aaf24391aeb1e
Parents: 6ec9289
Author: chitin <chitin1027@gmail.com>
Authored: Wed Nov 9 10:47:06 2016 +0800
Committer: wujinhu <wujinhu920@126.com>
Committed: Wed Nov 9 10:47:06 2016 +0800

----------------------------------------------------------------------
 .../jpm/mr/history/parser/JHFEventReaderBase.java     |  2 +-
 .../eagle/jpm/mr/running/MRRunningJobApplication.java |  3 ++-
 .../eagle/jpm/mr/running/parser/MRJobParser.java      |  8 ++++++--
 .../jpm/mr/running/storm/MRRunningJobParseBolt.java   |  8 ++++++--
 .../jpm/spark/history/crawl/JHFSparkEventReader.java  |  2 +-
 .../java/org/apache/eagle/jpm/util/Constants.java     |  4 ++++
 .../apache/eagle/jpm/util/JobNameNormalization.java   | 14 ++++----------
 7 files changed, 24 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
index e48370d..80cdb1c 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java
@@ -200,7 +200,7 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher
impl
             }
 
             if (jobDefId == null) {
-                this.jobDefId = JobNameNormalization.getInstance().normalize(jobName);
+                this.jobDefId = JobNameNormalization.getInstance(this.appConfig.getConfig()).normalize(jobName);
             } else {
                 LOG.debug("Got JobDefId from job configuration for " + id + ": " + jobDefId);
                 this.jobDefId = jobDefId;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
index bfdde13..e5c7c87 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/MRRunningJobApplication.java
@@ -66,7 +66,8 @@ public class MRRunningJobApplication extends StormApplication {
                 mrRunningJobConfig.getEagleServiceConfig(),
                 mrRunningJobConfig.getEndpointConfig(),
                 mrRunningJobConfig.getZkStateConfig(),
-                confKeyKeys),
+                confKeyKeys,
+                config),
                 tasks).setNumTasks(tasks).fieldsGrouping(spoutName, new Fields("appId"));
         return topologyBuilder.createTopology();
     }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
index 945e8ce..7b90dfc 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobParser.java
@@ -18,6 +18,7 @@
 
 package org.apache.eagle.jpm.mr.running.parser;
 
+import com.typesafe.config.Config;
 import org.apache.eagle.jpm.mr.running.MRRunningJobConfig;
 import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager;
 import org.apache.eagle.jpm.mr.runningentity.JobConfig;
@@ -79,6 +80,7 @@ public class MRJobParser implements Runnable {
     private static final int TOP_BOTTOM_TASKS_BY_ELAPSED_TIME = 10;
     private static final int FLUSH_TASKS_EVERY_TIME = 5;
     private static final int MAX_TASKS_PERMIT = 5000;
+    private Config config;
 
     static {
         OBJ_MAPPER.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
@@ -88,7 +90,8 @@ public class MRJobParser implements Runnable {
                        MRRunningJobConfig.EagleServiceConfig eagleServiceConfig,
                        AppInfo app, Map<String, JobExecutionAPIEntity> mrJobMap,
                        MRRunningJobManager runningJobManager, ResourceFetcher rmResourceFetcher,
-                       List<String> configKeys) {
+                       List<String> configKeys,
+                       Config config) {
         this.app = app;
         this.mrJobEntityMap = new HashMap<>();
         this.mrJobEntityMap = mrJobMap;
@@ -107,6 +110,7 @@ public class MRJobParser implements Runnable {
         this.rmResourceFetcher = rmResourceFetcher;
         this.finishedTaskIds = new HashSet<>();
         this.configKeys = configKeys;
+        this.config = config;
     }
 
     public void setAppInfo(AppInfo app) {
@@ -195,7 +199,7 @@ public class MRJobParser implements Runnable {
             jobExecutionAPIEntity.setTags(new HashMap<>(commonTags));
             jobExecutionAPIEntity.getTags().put(MRJobTagName.JOB_ID.toString(), id);
             jobExecutionAPIEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), mrJob.getName());
-            String jobDefId = JobNameNormalization.getInstance().normalize(mrJob.getName());
+            String jobDefId = JobNameNormalization.getInstance(this.config).normalize(mrJob.getName());
             jobExecutionAPIEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), jobDefId);
             if (mrJobConfigs.get(id) != null) {
                 JobConfig jobConfig = mrJobConfigs.get(id);

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java
b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java
index 9ebc1a7..e2767d8 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/storm/MRRunningJobParseBolt.java
@@ -18,6 +18,7 @@
 
 package org.apache.eagle.jpm.mr.running.storm;
 
+import com.typesafe.config.Config;
 import org.apache.eagle.jpm.mr.running.MRRunningJobConfig;
 import org.apache.eagle.jpm.mr.running.parser.MRJobParser;
 import org.apache.eagle.jpm.mr.running.recover.MRRunningJobManager;
@@ -49,16 +50,19 @@ public class MRRunningJobParseBolt extends BaseRichBolt {
     private MRRunningJobConfig.EagleServiceConfig eagleServiceConfig;
     private ResourceFetcher resourceFetcher;
     private List<String> configKeys;
+    private Config config;
 
     public MRRunningJobParseBolt(MRRunningJobConfig.EagleServiceConfig eagleServiceConfig,
                                  MRRunningJobConfig.EndpointConfig endpointConfig,
                                  MRRunningJobConfig.ZKStateConfig zkStateConfig,
-                                 List<String> configKeys) {
+                                 List<String> configKeys,
+                                 Config config) {
         this.eagleServiceConfig = eagleServiceConfig;
         this.endpointConfig = endpointConfig;
         this.runningMRParsers = new HashMap<>();
         this.zkStateConfig = zkStateConfig;
         this.configKeys = configKeys;
+        this.config = config;
     }
 
     @Override
@@ -79,7 +83,7 @@ public class MRRunningJobParseBolt extends BaseRichBolt {
         MRJobParser applicationParser;
         if (!runningMRParsers.containsKey(appInfo.getId())) {
             applicationParser = new MRJobParser(endpointConfig, eagleServiceConfig,
-                    appInfo, mrJobs, runningJobManager, this.resourceFetcher, configKeys);
+                    appInfo, mrJobs, runningJobManager, this.resourceFetcher, configKeys,
this.config);
             runningMRParsers.put(appInfo.getId(), applicationParser);
             LOG.info("create application parser for {}", appInfo.getId());
         } else {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
index 82b7607..a4ac77e 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/crawl/JHFSparkEventReader.java
@@ -674,7 +674,7 @@ public class JHFSparkEventReader {
         if (null != assignedName) {
             return assignedName;
         } else {
-            return JobNameNormalization.getInstance().normalize(jobName);
+            return JobNameNormalization.getInstance(this.config.getConfig()).normalize(jobName);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
index 73100a9..038618f 100644
--- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
+++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java
@@ -187,4 +187,8 @@ public class Constants {
 
     public static final String HADOOP_HISTORY_TOTAL_METRIC_FORMAT = "hadoop.%s.history.%s";
     public static final String HADOOP_HISTORY_MINUTE_METRIC_FORMAT = "hadoop.%s.history.minute.%s";
+
+    public static final String JOB_NAME_NORMALIZATION_RULES_KEY_DEFAULT = "^(.*)[0-9]{4}/[0-9]{2}/[0-9]{2}/[0-9]{2}$
=> "
+        + "$1~ ; ^(oozie:launcher):T=(.*):W=(.*):A=(.*):ID=(?:.*)$ => $1-$2-$3-$4~ ; ^(.*)([0-9]{10})$
=> $1~ ; "
+        + "^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$ => uuid~ ; ^(.*)(?:[0-9]{8}/[0-9]{2}_[0-9]{2})$
=> $1~";
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/ab3ed13e/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java
b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java
index b3673ea..0b38c67 100644
--- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java
+++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/JobNameNormalization.java
@@ -29,7 +29,6 @@ import java.util.regex.Pattern;
 
 public class JobNameNormalization {
     private static Logger LOG = LoggerFactory.getLogger(JobNameNormalization.class);
-    private static JobNameNormalization instance = new JobNameNormalization();
     private static final String JOB_NAME_NORMALIZATION_RULES_KEY = "job.name.normalization.rules.key";
     private static final String PARAMETERIZED_PREFIX = "\\$";
     private static final String MULTIPLE_RULE_DILIMITER = ";";
@@ -59,16 +58,11 @@ public class JobNameNormalization {
         String target;
     }
 
-    private JobNameNormalization() {
+    private JobNameNormalization(Config conf) {
         try {
             // load normalization rules
-            Config conf = ConfigFactory.load();
             String key = JOB_NAME_NORMALIZATION_RULES_KEY.toLowerCase();
-            String value = conf.getString(key);
-            if (value == null) {
-                LOG.info("no job name normalization rules are loaded");
-                return;
-            }
+            String value = conf.getString(key) != null ? conf.getString(key) : Constants.JOB_NAME_NORMALIZATION_RULES_KEY_DEFAULT;
             // multiple rules are concatenated with semicolon, i.e. ;
             String[] rules = value.split(MULTIPLE_RULE_DILIMITER);
             for (String rule : rules) {
@@ -82,8 +76,8 @@ public class JobNameNormalization {
         }
     }
 
-    public static JobNameNormalization getInstance() {
-        return instance;
+    public static JobNameNormalization getInstance(Config config) {
+        return new JobNameNormalization(config);
     }
 
     private void addRule(String rule) {


Mime
View raw message