Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 67BBB10C34 for ; Thu, 13 Feb 2014 00:09:15 +0000 (UTC) Received: (qmail 88106 invoked by uid 500); 13 Feb 2014 00:09:14 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 88049 invoked by uid 500); 13 Feb 2014 00:09:14 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 88041 invoked by uid 99); 13 Feb 2014 00:09:14 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 13 Feb 2014 00:09:14 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 13 Feb 2014 00:09:09 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 30A72238889B; Thu, 13 Feb 2014 00:08:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1567816 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ hadoop-yarn/hadoop-yarn-server/had... Date: Thu, 13 Feb 2014 00:08:48 -0000 To: yarn-commits@hadoop.apache.org From: zjshen@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140213000848.30A72238889B@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: zjshen Date: Thu Feb 13 00:08:47 2014 New Revision: 1567816 URL: http://svn.apache.org/r1567816 Log: YARN-1578. Fixed reading incomplete application attempt and container data in FileSystemApplicationHistoryStore. Contributed by Shinichi Yamashita. Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1567816&r1=1567815&r2=1567816&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Feb 13 00:08:47 2014 @@ -259,6 +259,9 @@ Release 2.4.0 - UNRELEASED YARN-1692. ConcurrentModificationException in fair scheduler AppSchedulable (Sangjin Lee via Sandy Ryza) + YARN-1578. Fixed reading incomplete application attempt and container data + in FileSystemApplicationHistoryStore. (Shinichi Yamashita via zjshen) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java?rev=1567816&r1=1567815&r2=1567816&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java Thu Feb 13 00:08:47 2014 @@ -215,17 +215,30 @@ public class FileSystemApplicationHistor getApplicationAttempts(ApplicationId appId) throws IOException { Map historyDataMap = new HashMap(); - Map> startFinshDataMap = - new HashMap>(); HistoryFileReader hfReader = getHistoryFileReader(appId); try { while (hfReader.hasNext()) { HistoryFileReader.Entry entry = hfReader.next(); - if (entry.key.id.startsWith(ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) { - if (entry.key.suffix.equals(START_DATA_SUFFIX)) { - retrieveStartFinishData(appId, entry, startFinshDataMap, true); - } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) { - retrieveStartFinishData(appId, entry, startFinshDataMap, false); + if (entry.key.id.startsWith( + ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) { + ApplicationAttemptId appAttemptId = + ConverterUtils.toApplicationAttemptId(entry.key.id); + if (appAttemptId.getApplicationId().equals(appId)) { + ApplicationAttemptHistoryData historyData = + historyDataMap.get(appAttemptId); + if (historyData == null) { + historyData = ApplicationAttemptHistoryData.newInstance( + appAttemptId, null, -1, null, null, null, + FinalApplicationStatus.UNDEFINED, null); + historyDataMap.put(appAttemptId, historyData); + } + if (entry.key.suffix.equals(START_DATA_SUFFIX)) { + mergeApplicationAttemptHistoryData(historyData, + parseApplicationAttemptStartData(entry.value)); + } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) { + mergeApplicationAttemptHistoryData(historyData, + parseApplicationAttemptFinishData(entry.value)); + } } } } @@ -237,45 +250,9 @@ public class FileSystemApplicationHistor } finally { hfReader.close(); } - for (Map.Entry> entry : startFinshDataMap - .entrySet()) { - ApplicationAttemptHistoryData historyData = - ApplicationAttemptHistoryData.newInstance(entry.getKey(), null, -1, - null, null, null, FinalApplicationStatus.UNDEFINED, null); - mergeApplicationAttemptHistoryData(historyData, - entry.getValue().startData); - mergeApplicationAttemptHistoryData(historyData, - entry.getValue().finishData); - historyDataMap.put(entry.getKey(), historyData); - } return historyDataMap; } - private - void - retrieveStartFinishData( - ApplicationId appId, - HistoryFileReader.Entry entry, - Map> startFinshDataMap, - boolean start) throws IOException { - ApplicationAttemptId appAttemptId = - ConverterUtils.toApplicationAttemptId(entry.key.id); - if (appAttemptId.getApplicationId().equals(appId)) { - StartFinishDataPair pair = - startFinshDataMap.get(appAttemptId); - if (pair == null) { - pair = - new StartFinishDataPair(); - startFinshDataMap.put(appAttemptId, pair); - } - if (start) { - pair.startData = parseApplicationAttemptStartData(entry.value); - } else { - pair.finishData = parseApplicationAttemptFinishData(entry.value); - } - } - } - @Override public ApplicationAttemptHistoryData getApplicationAttempt( ApplicationAttemptId appAttemptId) throws IOException { @@ -391,20 +368,30 @@ public class FileSystemApplicationHistor ApplicationAttemptId appAttemptId) throws IOException { Map historyDataMap = new HashMap(); - Map> startFinshDataMap = - new HashMap>(); HistoryFileReader hfReader = getHistoryFileReader(appAttemptId.getApplicationId()); try { while (hfReader.hasNext()) { HistoryFileReader.Entry entry = hfReader.next(); if (entry.key.id.startsWith(ConverterUtils.CONTAINER_PREFIX)) { - if (entry.key.suffix.equals(START_DATA_SUFFIX)) { - retrieveStartFinishData(appAttemptId, entry, startFinshDataMap, - true); - } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) { - retrieveStartFinishData(appAttemptId, entry, startFinshDataMap, - false); + ContainerId containerId = + ConverterUtils.toContainerId(entry.key.id); + if (containerId.getApplicationAttemptId().equals(appAttemptId)) { + ContainerHistoryData historyData = + historyDataMap.get(containerId); + if (historyData == null) { + historyData = ContainerHistoryData.newInstance( + containerId, null, null, null, Long.MIN_VALUE, + Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null); + historyDataMap.put(containerId, historyData); + } + if (entry.key.suffix.equals(START_DATA_SUFFIX)) { + mergeContainerHistoryData(historyData, + parseContainerStartData(entry.value)); + } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) { + mergeContainerHistoryData(historyData, + parseContainerFinishData(entry.value)); + } } } } @@ -416,43 +403,9 @@ public class FileSystemApplicationHistor } finally { hfReader.close(); } - for (Map.Entry> entry : startFinshDataMap - .entrySet()) { - ContainerHistoryData historyData = - ContainerHistoryData - .newInstance(entry.getKey(), null, null, null, Long.MIN_VALUE, - Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null); - mergeContainerHistoryData(historyData, entry.getValue().startData); - mergeContainerHistoryData(historyData, entry.getValue().finishData); - historyDataMap.put(entry.getKey(), historyData); - } return historyDataMap; } - private - void - retrieveStartFinishData( - ApplicationAttemptId appAttemptId, - HistoryFileReader.Entry entry, - Map> startFinshDataMap, - boolean start) throws IOException { - ContainerId containerId = ConverterUtils.toContainerId(entry.key.id); - if (containerId.getApplicationAttemptId().equals(appAttemptId)) { - StartFinishDataPair pair = - startFinshDataMap.get(containerId); - if (pair == null) { - pair = - new StartFinishDataPair(); - startFinshDataMap.put(containerId, pair); - } - if (start) { - pair.startData = parseContainerStartData(entry.value); - } else { - pair.finishData = parseContainerFinishData(entry.value); - } - } - } - @Override public void applicationStarted(ApplicationStartData appStart) throws IOException { @@ -828,14 +781,5 @@ public class FileSystemApplicationHistor id = in.readUTF(); suffix = in.readUTF(); } - } - - private static class StartFinishDataPair { - - private S startData; - private F finishData; - - } - } Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java?rev=1567816&r1=1567815&r2=1567816&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java Thu Feb 13 00:08:47 2014 @@ -72,6 +72,12 @@ public class TestFileSystemApplicationHi } private void testWriteHistoryData(int num) throws IOException { + testWriteHistoryData(num, false, false); + } + + private void testWriteHistoryData( + int num, boolean missingContainer, boolean missingApplicationAttempt) + throws IOException { // write application history data for (int i = 1; i <= num; ++i) { ApplicationId appId = ApplicationId.newInstance(0, i); @@ -83,21 +89,31 @@ public class TestFileSystemApplicationHi ApplicationAttemptId.newInstance(appId, j); writeApplicationAttemptStartData(appAttemptId); + if (missingApplicationAttempt && j == num) { + continue; + } // write container history data for (int k = 1; k <= num; ++k) { ContainerId containerId = ContainerId.newInstance(appAttemptId, k); writeContainerStartData(containerId); + if (missingContainer && k == num) { + continue; + } writeContainerFinishData(containerId); - - writeApplicationAttemptFinishData(appAttemptId); } + writeApplicationAttemptFinishData(appAttemptId); } - writeApplicationFinishData(appId); } } private void testReadHistoryData(int num) throws IOException { + testReadHistoryData(num, false, false); + } + + private void testReadHistoryData( + int num, boolean missingContainer, boolean missingApplicationAttempt) + throws IOException { // read application history data Assert.assertEquals(num, store.getAllApplications().size()); for (int i = 1; i <= num; ++i) { @@ -116,8 +132,14 @@ public class TestFileSystemApplicationHi store.getApplicationAttempt(appAttemptId); Assert.assertNotNull(attemptData); Assert.assertEquals(appAttemptId.toString(), attemptData.getHost()); - Assert.assertEquals(appAttemptId.toString(), - attemptData.getDiagnosticsInfo()); + + if (missingApplicationAttempt && j == num) { + Assert.assertNull(attemptData.getDiagnosticsInfo()); + continue; + } else { + Assert.assertEquals(appAttemptId.toString(), + attemptData.getDiagnosticsInfo()); + } // read container history data Assert.assertEquals(num, store.getContainers(appAttemptId).size()); @@ -127,8 +149,12 @@ public class TestFileSystemApplicationHi Assert.assertNotNull(containerData); Assert.assertEquals(Priority.newInstance(containerId.getId()), containerData.getPriority()); - Assert.assertEquals(containerId.toString(), - containerData.getDiagnosticsInfo()); + if (missingContainer && k == num) { + Assert.assertNull(containerData.getDiagnosticsInfo()); + } else { + Assert.assertEquals(containerId.toString(), + containerData.getDiagnosticsInfo()); + } } ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId); @@ -193,4 +219,15 @@ public class TestFileSystemApplicationHi Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20); } + @Test + public void testMissingContainerHistoryData() throws IOException { + testWriteHistoryData(3, true, false); + testReadHistoryData(3, true, false); + } + + @Test + public void testMissingApplicationAttemptHistoryData() throws IOException { + testWriteHistoryData(3, false, true); + testReadHistoryData(3, false, true); + } }