Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id AB73B200C22 for ; Tue, 21 Feb 2017 08:57:31 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id A9FD7160B68; Tue, 21 Feb 2017 07:57:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id CF4AB160B3E for ; Tue, 21 Feb 2017 08:57:29 +0100 (CET) Received: (qmail 47544 invoked by uid 500); 21 Feb 2017 07:57:28 -0000 Mailing-List: contact commits-help@eagle.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@eagle.apache.org Delivered-To: mailing list commits@eagle.apache.org Received: (qmail 47535 invoked by uid 99); 21 Feb 2017 07:57:28 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Feb 2017 07:57:28 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C8876E004D; Tue, 21 Feb 2017 07:57:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jinhuwu@apache.org To: commits@eagle.apache.org Message-Id: <83b1ef70e6e949398198e7cbf5cae8b4@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: eagle git commit: [EAGLE-917] add Job error category and error message mapping Date: Tue, 21 Feb 2017 07:57:28 +0000 (UTC) archived-at: Tue, 21 Feb 2017 07:57:31 -0000 Repository: eagle Updated Branches: refs/heads/master 4a5c4a43c -> 45a649399 [EAGLE-917] add Job error category and error message mapping Author: wujinhu Closes #827 from wujinhu/EAGLE-917. Project: http://git-wip-us.apache.org/repos/asf/eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/45a64939 Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/45a64939 Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/45a64939 Branch: refs/heads/master Commit: 45a6493999853328b89c16cd1dbc462a4315953a Parents: 4a5c4a4 Author: wujinhu Authored: Tue Feb 21 15:57:20 2017 +0800 Committer: wujinhu Committed: Tue Feb 21 15:57:20 2017 +0800 ---------------------------------------------------------------------- .../mr/MRMetricsAggregateContainer.java | 3 +- .../UnExpectedLongDurationJobProcessor.java | 2 +- .../mr/historyentity/JPAEntityRepository.java | 1 + .../JobConfigurationAPIEntity.java | 2 +- .../jpm/mr/historyentity/JobCountEntity.java | 2 +- .../JobErrorCategoryMappingAPIEntity.java | 62 ++++++++++++++++++++ .../jpm/mr/historyentity/JobEventAPIEntity.java | 2 +- .../mr/historyentity/JobExecutionAPIEntity.java | 4 +- .../JobProcessTimeStampEntity.java | 2 +- .../historyentity/JobSuggestionAPIEntity.java | 4 +- .../TaskAttemptCounterAPIEntity.java | 2 +- .../TaskAttemptErrorCategoryEntity.java | 2 +- .../TaskAttemptExecutionAPIEntity.java | 5 +- .../historyentity/TaskExecutionAPIEntity.java | 2 +- .../TaskFailureCountAPIEntity.java | 2 +- .../mr/runningentity/JobExecutionAPIEntity.java | 2 +- .../TaskAttemptExecutionAPIEntity.java | 2 +- .../runningentity/TaskExecutionAPIEntity.java | 2 +- .../MRHistoryJobApplicationHealthCheck.java | 2 +- .../mr/history/MRHistoryJobDailyReporter.java | 8 +-- .../mr/history/parser/JHFEventReaderBase.java | 30 ++++++++++ .../service/jpm/MRJobExecutionResource.java | 16 ++--- .../service/jpm/MRTaskExecutionResource.java | 12 ++-- .../eagle/service/jpm/RunningQueueResource.java | 8 +-- .../org/apache/eagle/jpm/util/Constants.java | 29 ++++----- 25 files changed, 150 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java index dd1980b..be139fb 100644 --- a/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java +++ b/eagle-jpm/eagle-jpm-aggregation/src/main/java/org/apache/eagle/jpm/aggregation/mr/MRMetricsAggregateContainer.java @@ -21,7 +21,6 @@ package org.apache.eagle.jpm.aggregation.mr; import org.apache.eagle.jpm.aggregation.AggregationConfig; import org.apache.eagle.jpm.aggregation.common.MetricAggregator; import org.apache.eagle.jpm.aggregation.common.MetricsAggregateContainer; -import org.apache.eagle.jpm.mr.historyentity.JobProcessTimeStampEntity; import org.apache.eagle.jpm.util.Constants; import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; import org.apache.eagle.service.client.IEagleServiceClient; @@ -62,7 +61,7 @@ public class MRMetricsAggregateContainer implements MetricsAggregateContainer, S eagleServiceConfig.password); String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}", - Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME, + Constants.MR_JOB_PROCESS_TIME_STAMP_NAME, appConfig.getStormConfig().site); GenericServiceAPIResponseEntity response = client http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java index 8f655ba..f7748f8 100644 --- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java +++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/sla/processors/UnExpectedLongDurationJobProcessor.java @@ -90,7 +90,7 @@ public class UnExpectedLongDurationJobProcessor implements Processor, Serializab } String query = String.format("%s[@site=\"%s\" and @jobDefId=\"%s\"]<@site>{avg(durationTime)}", - org.apache.eagle.jpm.util.Constants.JPA_JOB_EXECUTION_SERVICE_NAME, + org.apache.eagle.jpm.util.Constants.MR_JOB_EXECUTION_SERVICE_NAME, mrJobAnalysisEntity.getSiteId(), URLEncoder.encode(mrJobAnalysisEntity.getJobDefId())); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java index 8c65adf..54383bd 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JPAEntityRepository.java @@ -30,6 +30,7 @@ public class JPAEntityRepository extends EntityRepository { entitySet.add(JobConfigurationAPIEntity.class); entitySet.add(JobEventAPIEntity.class); entitySet.add(JobExecutionAPIEntity.class); + entitySet.add(JobErrorCategoryMappingAPIEntity.class); entitySet.add(TaskAttemptExecutionAPIEntity.class); entitySet.add(TaskExecutionAPIEntity.class); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java index 6298fe8..36875e3 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigurationAPIEntity.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa") @ColumnFamily("f") @Prefix("jconf") -@Service(Constants.JPA_JOB_CONFIG_SERVICE_NAME) +@Service(Constants.MR_JOB_CONFIG_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java index 5b4163a..0fde454 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobCountEntity.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejcount") @ColumnFamily("f") @Prefix("jcount") -@Service(Constants.JPA_JOB_COUNT_SERVICE_NAME) +@Service(Constants.MR_JOB_COUNT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class JobCountEntity extends TaggedLogAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java new file mode 100644 index 0000000..91c2c68 --- /dev/null +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobErrorCategoryMappingAPIEntity.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.eagle.jpm.mr.historyentity; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.eagle.jpm.util.Constants; +import org.apache.eagle.log.entity.meta.*; + +import java.util.List; + +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@Table("eaglejpa") +@ColumnFamily("f") +@Prefix("jecm") +@Service(Constants.MR_JOB_ERROR_MAPPING_SERVICE_NAME) +@TimeSeries(true) +@Partition({"site"}) +@Indexes({ + @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false), + @Index(name = "Index_1_jobDefId", columns = { "jobDefId" }, unique = false), + @Index(name = "Index_1_jobIdAndErrorCategory", columns = { "jobId", "errorCategory" }, unique = true) + }) +public class JobErrorCategoryMappingAPIEntity extends JobBaseAPIEntity { + @Column("a") + private String error; + @Column("b") + private List taskAttempts; + + public String getError() { + return error; + } + + public void setError(String error) { + this.error = error; + valueChanged("error"); + } + + public List getTaskAttempts() { + return taskAttempts; + } + + public void setTaskAttempts(List taskAttempts) { + this.taskAttempts = taskAttempts; + valueChanged("taskAttempts"); + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java index aef3750..56f541b 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobEventAPIEntity.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa") @ColumnFamily("f") @Prefix("jevent") -@Service(Constants.JPA_JOB_EVENT_SERVICE_NAME) +@Service(Constants.MR_JOB_EVENT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class JobEventAPIEntity extends JobBaseAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java index df7b271..746843d 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java @@ -23,13 +23,11 @@ import org.apache.eagle.jpm.util.jobcounter.JobCounters; import org.apache.eagle.log.entity.meta.*; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.util.Map; - @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @Table("eaglejpa") @ColumnFamily("f") @Prefix("jexec") -@Service(Constants.JPA_JOB_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_JOB_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java index ee632f4..1490390 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobProcessTimeStampEntity.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_process") @ColumnFamily("f") @Prefix("process") -@Service(Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME) +@Service(Constants.MR_JOB_PROCESS_TIME_STAMP_NAME) @TimeSeries(true) @Partition({"site"}) public class JobProcessTimeStampEntity extends TaggedLogAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java index 3863a5d..e433672 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobSuggestionAPIEntity.java @@ -23,13 +23,13 @@ import org.apache.eagle.log.entity.meta.*; import java.util.List; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import static org.apache.eagle.jpm.util.Constants.JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME; @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) @Table("eaglejpa") @ColumnFamily("f") @Prefix("jsuggestion") -@Service(JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME) +@Service(MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java index 04fe7d7..33a5102 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptCounterAPIEntity.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_anomaly") @ColumnFamily("f") @Prefix("tacount") -@Service(Constants.JPA_TASK_ATTEMPT_COUNTER_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_COUNTER_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class TaskAttemptCounterAPIEntity extends JobBaseAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java index 37f6451..1228f4e 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptErrorCategoryEntity.java @@ -28,7 +28,7 @@ import org.apache.eagle.log.entity.meta.*; @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("taexece") -@Service(Constants.JPA_TASK_ATTEMPT_ERROR_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_ERROR_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java index 46fcf5e..b4bb916 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskAttemptExecutionAPIEntity.java @@ -27,11 +27,12 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("taexec") -@Service(Constants.JPA_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ - @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false) + @Index(name = "Index_1_jobId", columns = { "jobId" }, unique = false), + @Index(name = "Index_1_taskAttemptId", columns = { "taskAttemptId" }, unique = true) }) public class TaskAttemptExecutionAPIEntity extends JobBaseAPIEntity { @Column("a") http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java index 9600fe2..9f85555 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskExecutionAPIEntity.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_task") @ColumnFamily("f") @Prefix("texec") -@Service(Constants.JPA_TASK_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_TASK_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java index 3e5e997..f82bcaf 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/TaskFailureCountAPIEntity.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eaglejpa_anomaly") @ColumnFamily("f") @Prefix("taskfailurecount") -@Service(Constants.JPA_TASK_FAILURE_COUNT_SERVICE_NAME) +@Service(Constants.MR_TASK_FAILURE_COUNT_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) public class TaskFailureCountAPIEntity extends JobBaseAPIEntity { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java index 082612b..3327a18 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/JobExecutionAPIEntity.java @@ -29,7 +29,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eagleMRRunningJobs") @ColumnFamily("f") @Prefix("jobs") -@Service(Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java index 0d1703a..4fa7775 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskAttemptExecutionAPIEntity.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eagleMRRunningTasks") @ColumnFamily("f") @Prefix("tasks_exec_attempt") -@Service(Constants.JPA_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java index 61c92c1..e9f0fcf 100644 --- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java +++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/runningentity/TaskExecutionAPIEntity.java @@ -28,7 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; @Table("eagleMRRunningTasks") @ColumnFamily("f") @Prefix("tasks_exec") -@Service(Constants.JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME) +@Service(Constants.MR_RUNNING_TASK_EXECUTION_SERVICE_NAME) @TimeSeries(true) @Partition({"site"}) @Indexes({ http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java index 4016e6d..90544e7 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java @@ -61,7 +61,7 @@ public class MRHistoryJobApplicationHealthCheck extends ApplicationHealthCheckBa } String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}", - Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME, + Constants.MR_JOB_PROCESS_TIME_STAMP_NAME, mrHistoryJobConfig.getJobHistoryEndpointConfig().site); GenericServiceAPIResponseEntity response = client http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java index 9aef640..541d352 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java @@ -208,16 +208,16 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService { private Map buildJobSummery(String site, long startTime, long endTime) { Map data = new HashMap<>(); - String query = String.format(STATUS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String query = String.format(STATUS_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); Map jobSummery = queryGroupByMetrics(query, startTime, endTime, Integer.MAX_VALUE); if (jobSummery == null || jobSummery.isEmpty()) { LOG.warn("Result set is empty for query={}", query); return data; } Long totalJobs = jobSummery.values().stream().reduce((a, b) -> a + b).get(); - String finishedJobQuery = String.format(FINISHED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); - String failedJobQuery = String.format(FAILED_JOBS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); - String succeededJobQuery = String.format(SUCCEEDED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, jobOvertimeLimit * DateTimeUtil.ONEHOUR, endTime); + String finishedJobQuery = String.format(FINISHED_JOB_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String failedJobQuery = String.format(FAILED_JOBS_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, endTime); + String succeededJobQuery = String.format(SUCCEEDED_JOB_QUERY, Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, jobOvertimeLimit * DateTimeUtil.ONEHOUR, endTime); data.put(SUMMARY_INFO_KEY, processResult(jobSummery, totalJobs)); data.put(FAILED_JOB_USERS_KEY, buildJobSummery(failedJobQuery, startTime, endTime, jobSummery.get(Constants.JobState.FAILED.toString()))); data.put(SUCCEEDED_JOB_USERS_KEY, buildJobSummery(succeededJobQuery, startTime, endTime, jobSummery.get(Constants.JobState.SUCCEEDED.toString()))); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java index d58eadc..afcd7be 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/parser/JHFEventReaderBase.java @@ -57,6 +57,9 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl // taskAttemptID to task attempt startTime protected Map taskAttemptStartTime; + //errorCategory, + protected Map> errorCategoryTaskMapping; + // taskID to host mapping, for task it's the host where the last attempt runs on protected Map taskRunningHosts; // hostname to rack mapping @@ -117,6 +120,7 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl taskStartTime = new HashMap<>(); taskAttemptStartTime = new HashMap<>(); + errorCategoryTaskMapping = new HashMap<>(); this.configuration = configuration; @@ -305,6 +309,25 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl formatDiagnostics(values.get(Keys.DIAGNOSTICS)); entityCreated(jobExecutionEntity); + for (String errorCategory : errorCategoryTaskMapping.keySet()) { + JobErrorCategoryMappingAPIEntity jobErrorCategoryMappingAPIEntity = new JobErrorCategoryMappingAPIEntity(); + jobErrorCategoryMappingAPIEntity.setTags(new HashMap<>(jobExecutionEntity.getTags())); + jobErrorCategoryMappingAPIEntity.getTags().put(MRJobTagName.ERROR_CATEGORY.toString(), errorCategory); + jobErrorCategoryMappingAPIEntity.setTimestamp(jobExecutionEntity.getTimestamp()); + + for (String taskId : errorCategoryTaskMapping.get(errorCategory).keySet()) { + jobErrorCategoryMappingAPIEntity.getTaskAttempts().add(errorCategoryTaskMapping.get(errorCategory).get(taskId)); + } + + if (jobErrorCategoryMappingAPIEntity.getTaskAttempts().size() > 0) { + jobErrorCategoryMappingAPIEntity.setError( + attempt2ErrorMsg.get( + jobErrorCategoryMappingAPIEntity.getTaskAttempts().get(0) + ).getRight() + ); + } + entityCreated(jobErrorCategoryMappingAPIEntity); + } if (configuration != null && totalCounters != null) { JobCounters parsedTotalCounters = parseCounters(totalCounters); JobCounters parsedMapCounters = parseCounters(mapCounters); @@ -491,6 +514,13 @@ public abstract class JHFEventReaderBase extends JobEntityCreationPublisher impl taskAttemptErrorCategoryEntity.setEndTime(entity.getEndTime()); taskAttemptErrorCategoryEntity.setTimestamp(entity.getTimestamp()); entityCreated(taskAttemptErrorCategoryEntity); + + String errorCategory = entity.getTags().get(MRJobTagName.ERROR_CATEGORY.toString()); + if (!errorCategoryTaskMapping.containsKey(errorCategory)) { + errorCategoryTaskMapping.put(errorCategory, new HashMap<>()); + } + + errorCategoryTaskMapping.get(errorCategory).put(taskID, taskAttemptID); } taskAttemptStartTime.remove(taskAttemptID); } else { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java index 28e6bb3..5804e0c 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java @@ -69,7 +69,7 @@ public class MRJobExecutionResource { StopWatch stopWatch = new StopWatch(); stopWatch.start(); - String jobQuery = String.format(query, Constants.JPA_JOB_EXECUTION_SERVICE_NAME); + String jobQuery = String.format(query, Constants.MR_JOB_EXECUTION_SERVICE_NAME); GenericServiceAPIResponseEntity res = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose); if (res.isSuccess() && res.getObj() != null) { @@ -80,7 +80,7 @@ public class MRJobExecutionResource { jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString())); } } - jobQuery = String.format(query, Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME); + jobQuery = String.format(query, Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME); GenericServiceAPIResponseEntity runningRes = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose); if (runningRes.isSuccess() && runningRes.getObj() != null) { @@ -150,7 +150,7 @@ public class MRJobExecutionResource { StopWatch stopWatch = new StopWatch(); stopWatch.start(); String queryFormat = "%s[%s]{*}"; - String queryString = String.format(queryFormat, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, condition); + String queryString = String.format(queryFormat, Constants.MR_JOB_EXECUTION_SERVICE_NAME, condition); GenericServiceAPIResponseEntity res = ResourceUtils.getQueryResult(queryString, null, null); if (res.isSuccess() && res.getObj() != null) { for (TaggedLogAPIEntity o : res.getObj()) { @@ -158,7 +158,7 @@ public class MRJobExecutionResource { jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString())); } } - queryString = String.format(queryFormat, Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, condition); + queryString = String.format(queryFormat, Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, condition); res = ResourceUtils.getQueryResult(queryString, null, null); if (res.isSuccess() && res.getObj() != null) { for (TaggedLogAPIEntity o : res.getObj()) { @@ -214,13 +214,13 @@ public class MRJobExecutionResource { response.errMessage = e.getMessage(); return response; } - String query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site); + String query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.MR_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity historyRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { response.errMessage = String.format("Catch an exception during fetch history jobs: %s with query=%s", historyRes.getException(), query); return response; } - query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, site); + query = String.format("%s[@site=\"%s\"]{@startTime,@endTime,@jobType,@jobId}", Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity runningRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!runningRes.isSuccess() || runningRes.getObj() == null) { response.errMessage = String.format("Catch an exception during fetch running jobs: %s with query=%s", runningRes.getException(), query); @@ -289,7 +289,7 @@ public class MRJobExecutionResource { return response; } String query = String.format("%s[@site=\"%s\" AND @startTime<=\"%s\" AND @endTime>=\"%s\"]{@startTime,@endTime}", - Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, timePointsInMills, timePointsInMills); + Constants.MR_JOB_EXECUTION_SERVICE_NAME, site, timePointsInMills, timePointsInMills); GenericServiceAPIResponseEntity historyRes = ResourceUtils.getQueryResult(query, searchStartTime, searchEndTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { return historyRes; @@ -333,7 +333,7 @@ public class MRJobExecutionResource { response.errMessage = "IllegalArgument: site, startTime, endTime, or timeDistInSecs is null"; return response; } - String query = String.format("%s[@site=\"%s\"]{@durationTime,@jobType}", Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site); + String query = String.format("%s[@site=\"%s\"]{@durationTime,@jobType}", Constants.MR_JOB_EXECUTION_SERVICE_NAME, site); GenericServiceAPIResponseEntity historyRes = ResourceUtils.getQueryResult(query, startTime, endTime); if (!historyRes.isSuccess() || historyRes.getObj() == null) { response.errMessage = String.format("Catch an exception: %s with query=%s", historyRes.getException(), query); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java index 1125387..d81e7d9 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRTaskExecutionResource.java @@ -60,7 +60,7 @@ public class MRTaskExecutionResource { List finishedTaskCount = new ArrayList<>(); List times = ResourceUtils.parseDistributionList(timeDistInSecs); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, jobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity historyRes = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (historyRes.isSuccess() && historyRes.getObj() != null && historyRes.getObj().size() > 0) { @@ -72,7 +72,7 @@ public class MRTaskExecutionResource { counter.entities.add(o); } } else { - query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME, site, jobId); + query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_RUNNING_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity runningRes = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (runningRes.isSuccess() && runningRes.getObj() != null) { @@ -105,7 +105,7 @@ public class MRTaskExecutionResource { @QueryParam("shortJob_id") String shortDurationJobId, @QueryParam("longJob_id") String longDurationJobId) { MRTaskExecutionResponse.TaskGroupResponse result = new MRTaskExecutionResponse.TaskGroupResponse(); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, shortDurationJobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, shortDurationJobId); GenericServiceAPIResponseEntity smallResponse = ResourceUtils.getQueryResult(query, null, null); if (!smallResponse.isSuccess() || smallResponse.getObj() == null) { result.errMessage = smallResponse.getException(); @@ -117,7 +117,7 @@ public class MRTaskExecutionResource { longestDuration = entity.getDuration(); } } - query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, longDurationJobId); + query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{*}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, longDurationJobId); GenericServiceAPIResponseEntity largeResponse = ResourceUtils.getQueryResult(query, null, null); if (!largeResponse.isSuccess() || largeResponse.getObj() == null) { result.errMessage = largeResponse.getException(); @@ -193,7 +193,7 @@ public class MRTaskExecutionResource { return result; } - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{@startTime,@endTime,@taskType}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, site, jobId); + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\"]{@startTime,@endTime,@taskType}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId); GenericServiceAPIResponseEntity response = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (!response.isSuccess() || response.getObj() == null) { result.errMessage = response.getException(); @@ -221,7 +221,7 @@ public class MRTaskExecutionResource { @PathParam("counterName") String counterName, @QueryParam("distRange") String distRange) { MRTaskExecutionResponse.TaskDistributionResponse result = new MRTaskExecutionResponse.TaskDistributionResponse(); - String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\" AND @taskType=\"%s\"]{@jobCounters}", Constants.JPA_TASK_EXECUTION_SERVICE_NAME, + String query = String.format("%s[@site=\"%s\" AND @jobId=\"%s\" AND @taskType=\"%s\"]{@jobCounters}", Constants.MR_TASK_EXECUTION_SERVICE_NAME, site, jobId, Constants.TaskType.MAP.toString()); GenericServiceAPIResponseEntity response = ResourceUtils.getQueryResult(query, jobStartTime, jobEndTime); if (!response.isSuccess() || response.getObj() == null) { http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java index 2632423..2b02052 100644 --- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java +++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/RunningQueueResource.java @@ -31,8 +31,8 @@ import java.text.ParseException; import java.util.*; import static org.apache.eagle.hadoop.queue.common.HadoopClusterConstants.QUEUE_MAPPING_SERVICE_NAME; -import static org.apache.eagle.jpm.util.Constants.JPA_JOB_EXECUTION_SERVICE_NAME; -import static org.apache.eagle.jpm.util.Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_JOB_EXECUTION_SERVICE_NAME; +import static org.apache.eagle.jpm.util.Constants.MR_RUNNING_JOB_EXECUTION_SERVICE_NAME; import static org.apache.eagle.jpm.util.MRJobTagName.JOB_ID; import static org.apache.eagle.jpm.util.MRJobTagName.JOB_QUEUE; import static org.apache.eagle.jpm.util.MRJobTagName.USER; @@ -87,7 +87,7 @@ public class RunningQueueResource { private List getRunningJobs(String site, long currentTime, String startTime, String endTime) throws Exception { GenericEntityServiceResource resource = new GenericEntityServiceResource(); String query = String.format("%s[@site=\"%s\" and @startTime<=%s and (@internalState=\"RUNNING\" or @endTime>%s)]{@jobId, @user, @queue, @allocatedMB}", - JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); + MR_RUNNING_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); GenericServiceAPIResponseEntity runningJobResponse = resource.search(query, startTime, endTime, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); if (!runningJobResponse.isSuccess() || runningJobResponse.getObj() == null) { @@ -102,7 +102,7 @@ public class RunningQueueResource { String startTime, String endTime) throws Exception { GenericEntityServiceResource resource = new GenericEntityServiceResource(); - String query = String.format("%s[@site=\"%s\" and @startTime<=%s and @endTime>%s]{@jobId}", JPA_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); + String query = String.format("%s[@site=\"%s\" and @startTime<=%s and @endTime>%s]{@jobId}", MR_JOB_EXECUTION_SERVICE_NAME, site, currentTime, currentTime); GenericServiceAPIResponseEntity response = resource.search(query, startTime, endTime, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null, false); http://git-wip-us.apache.org/repos/asf/eagle/blob/45a64939/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java index 4ee58a1..b87c41d 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/Constants.java @@ -104,20 +104,21 @@ public class Constants { public static final String TASK_FINISHED = "FINISHED"; //MR - public static final String JPA_JOB_CONFIG_SERVICE_NAME = "JobConfigService"; - public static final String JPA_JOB_EVENT_SERVICE_NAME = "JobEventService"; - public static final String JPA_JOB_EXECUTION_SERVICE_NAME = "JobExecutionService"; - public static final String JPA_JOB_COUNT_SERVICE_NAME = "JobCountService"; - public static final String JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME = "RunningJobExecutionService"; - public static final String JPA_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "TaskAttemptExecutionService"; - public static final String JPA_TASK_ATTEMPT_ERROR_SERVICE_NAME = "TaskAttemptErrorCategoryService"; - public static final String JPA_TASK_FAILURE_COUNT_SERVICE_NAME = "TaskFailureCountService"; - public static final String JPA_TASK_ATTEMPT_COUNTER_SERVICE_NAME = "TaskAttemptCounterService"; - public static final String JPA_TASK_EXECUTION_SERVICE_NAME = "TaskExecutionService"; - public static final String JPA_RUNNING_TASK_EXECUTION_SERVICE_NAME = "RunningTaskExecutionService"; - public static final String JPA_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "RunningTaskAttemptExecutionService"; - public static final String JPA_JOB_PROCESS_TIME_STAMP_NAME = "JobProcessTimeStampService"; - public static final String JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME = "JobOptimizerSuggestionService"; + public static final String MR_JOB_CONFIG_SERVICE_NAME = "JobConfigService"; + public static final String MR_JOB_EVENT_SERVICE_NAME = "JobEventService"; + public static final String MR_JOB_EXECUTION_SERVICE_NAME = "JobExecutionService"; + public static final String MR_JOB_ERROR_MAPPING_SERVICE_NAME = "JobErrorMappingService"; + public static final String MR_JOB_COUNT_SERVICE_NAME = "JobCountService"; + public static final String MR_RUNNING_JOB_EXECUTION_SERVICE_NAME = "RunningJobExecutionService"; + public static final String MR_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "TaskAttemptExecutionService"; + public static final String MR_TASK_ATTEMPT_ERROR_SERVICE_NAME = "TaskAttemptErrorCategoryService"; + public static final String MR_TASK_FAILURE_COUNT_SERVICE_NAME = "TaskFailureCountService"; + public static final String MR_TASK_ATTEMPT_COUNTER_SERVICE_NAME = "TaskAttemptCounterService"; + public static final String MR_TASK_EXECUTION_SERVICE_NAME = "TaskExecutionService"; + public static final String MR_RUNNING_TASK_EXECUTION_SERVICE_NAME = "RunningTaskExecutionService"; + public static final String MR_RUNNING_TASK_ATTEMPT_EXECUTION_SERVICE_NAME = "RunningTaskAttemptExecutionService"; + public static final String MR_JOB_PROCESS_TIME_STAMP_NAME = "JobProcessTimeStampService"; + public static final String MR_JOB_OPTIMIZER_SUGGESTION_SERVICE_NAME = "JobOptimizerSuggestionService"; public static final String JOB_TASK_TYPE_TAG = "taskType";