Return-Path: 
 <common-commits-return-44388-apmail-hadoop-common-commits-archive=hadoop.apache.org@hadoop.apache.org>
X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org
Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id EC7E11829C
	for <apmail-hadoop-common-commits-archive@www.apache.org>;
 Tue, 15 Dec 2015 09:05:32 +0000 (UTC)
Received: (qmail 41024 invoked by uid 500); 15 Dec 2015 09:05:32 -0000
Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org
Received: (qmail 40963 invoked by uid 500); 15 Dec 2015 09:05:32 -0000
Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:common-commits-help@hadoop.apache.org>
List-Unsubscribe: <mailto:common-commits-unsubscribe@hadoop.apache.org>
List-Post: <mailto:common-commits@hadoop.apache.org>
List-Id: <common-commits.hadoop.apache.org>
Reply-To: common-dev@hadoop.apache.org
Delivered-To: mailing list common-commits@hadoop.apache.org
Received: (qmail 40949 invoked by uid 99); 15 Dec 2015 09:05:32 -0000
Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org)
 (140.211.11.23)
    by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Dec 2015 09:05:32 +0000
Received: by git1-us-west.apache.org (ASF Mail Server at
 git1-us-west.apache.org, from userid 33)
	id 90CCDDFF6F; Tue, 15 Dec 2015 09:05:32 +0000 (UTC)
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: zxu@apache.org
To: common-commits@hadoop.apache.org
Message-Id: <cf7d26b224404a59af0247d0f52e631b@git.apache.org>
X-Mailer: ASF-Git Admin Mailer
Subject: hadoop git commit: MAPREDUCE-6436. JobHistory cache issue.
 Contributed by Kai Sasaki
Date: Tue, 15 Dec 2015 09:05:32 +0000 (UTC)

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 15124c86d -> da7bde60c


MAPREDUCE-6436. JobHistory cache issue. Contributed by Kai Sasaki

(cherry picked from commit 5b7078d06921893200163a3d29c8901c3c0107cb)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da7bde60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da7bde60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da7bde60

Branch: refs/heads/branch-2
Commit: da7bde60c0ca0157d06a48923b190e54550d81b2
Parents: 15124c8
Author: Zhihai Xu <zxu@apache.org>
Authored: Tue Dec 15 00:58:23 2015 -0800
Committer: Zhihai Xu <zxu@apache.org>
Committed: Tue Dec 15 01:03:10 2015 -0800

----------------------------------------------------------------------
 .../mapreduce/v2/hs/HistoryFileManager.java     | 37 ++++++++++++++++++--
 hadoop-yarn-project/CHANGES.txt                 |  2 ++
 2 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/da7bde60/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index b539072..01fb9d5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -219,13 +219,21 @@ public class HistoryFileManager extends AbstractService {
         // keeping the cache size exactly at the maximum.
         Iterator<JobId> keys = cache.navigableKeySet().iterator();
         long cutoff = System.currentTimeMillis() - maxAge;
+
+        // MAPREDUCE-6436: In order to reduce the number of logs written
+        // in case of a lot of move pending histories.
+        JobId firstInIntermediateKey = null;
+        int inIntermediateCount = 0;
+        JobId firstMoveFailedKey = null;
+        int moveFailedCount = 0;
+
         while(cache.size() > maxSize && keys.hasNext()) {
           JobId key = keys.next();
           HistoryFileInfo firstValue = cache.get(key);
           if(firstValue != null) {
             synchronized(firstValue) {
               if (firstValue.isMovePending()) {
-                if(firstValue.didMoveFail() && 
+                if(firstValue.didMoveFail() &&
                     firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
                   cache.remove(key);
                   //Now lets try to delete it
@@ -236,8 +244,17 @@ public class HistoryFileManager extends AbstractService {
                     		" that could not be moved to done.", e);
                   }
                 } else {
-                  LOG.warn("Waiting to remove " + key
-                      + " from JobListCache because it is not in done yet.");
+                  if (firstValue.didMoveFail()) {
+                    if (moveFailedCount == 0) {
+                      firstMoveFailedKey = key;
+                    }
+                    moveFailedCount += 1;
+                  } else {
+                    if (inIntermediateCount == 0) {
+                      firstInIntermediateKey = key;
+                    }
+                    inIntermediateCount += 1;
+                  }
                 }
               } else {
                 cache.remove(key);
@@ -245,6 +262,20 @@ public class HistoryFileManager extends AbstractService {
             }
           }
         }
+        // Log output only for first jobhisotry in pendings to restrict
+        // the total number of logs.
+        if (inIntermediateCount > 0) {
+          LOG.warn("Waiting to remove IN_INTERMEDIATE state histories " +
+                  "(e.g. " + firstInIntermediateKey + ") from JobListCache " +
+                  "because it is not in done yet. Total count is " +
+                  inIntermediateCount + ".");
+        }
+        if (moveFailedCount > 0) {
+          LOG.warn("Waiting to remove MOVE_FAILED state histories " +
+                  "(e.g. " + firstMoveFailedKey + ") from JobListCache " +
+                  "because it is not in done yet. Total count is " +
+                  moveFailedCount + ".");
+        }
       }
       return old;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/da7bde60/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3e3d746..64dabfd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1984,6 +1984,8 @@ Release 2.6.4 - UNRELEASED
 
   IMPROVEMENTS
 
+  MAPREDUCE-6436. JobHistory cache issue. (Kai Sasaki via zxu)
+
   OPTIMIZATIONS
 
   BUG FIXES