Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4C7D4200D50 for ; Thu, 26 Oct 2017 08:34:37 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4AB06160BF2; Thu, 26 Oct 2017 06:34:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id DCA851609E8 for ; Thu, 26 Oct 2017 08:34:33 +0200 (CEST) Received: (qmail 75647 invoked by uid 500); 26 Oct 2017 06:34:32 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 75638 invoked by uid 99); 26 Oct 2017 06:34:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Oct 2017 06:34:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A4861DFB77; Thu, 26 Oct 2017 06:34:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aajisaka@apache.org To: common-commits@hadoop.apache.org Date: Thu, 26 Oct 2017 06:34:32 -0000 Message-Id: <548655de607c4be2ae3b7a678b2c25a3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/5] hadoop git commit: Revert "HADOOP-14840. Tool to estimate resource requirements of an application pipeline based on prior executions. (Rui Li via Subru)." archived-at: Thu, 26 Oct 2017 06:34:37 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2 da7f989d8 -> 13f1a7e0e http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/api/TestJobMetaData.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/api/TestJobMetaData.java b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/api/TestJobMetaData.java deleted file mode 100644 index 69ba480..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/api/TestJobMetaData.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.api; - -import java.text.ParseException; - -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.translator.impl.LogParserUtil; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * Test JobMetaData. - */ -public class TestJobMetaData { - /** - * TODO: parametrize this test. - */ - private LogParserUtil logParserUtil = new LogParserUtil(); - - private JobMetaData jobMetaData; - private RecurrenceId recurrenceId; - - @Before public final void setup() throws ParseException { - recurrenceId = new RecurrenceId("Fraud Detection", "17/07/16 16:27:25"); - jobMetaData = new JobMetaData( - logParserUtil.stringToUnixTimestamp("17/07/16 16:27:25")); - jobMetaData.setRecurrenceId(recurrenceId); - jobMetaData.setContainerStart("C1", - logParserUtil.stringToUnixTimestamp("17/07/16 16:27:30")); - jobMetaData.setContainerEnd("C1", - logParserUtil.stringToUnixTimestamp("17/07/16 16:37:30")); - jobMetaData.setContainerStart("C2", - logParserUtil.stringToUnixTimestamp("17/07/16 16:27:40")); - jobMetaData.setContainerEnd("C2", - logParserUtil.stringToUnixTimestamp("17/07/16 16:37:40")); - jobMetaData.setJobFinishTime( - logParserUtil.stringToUnixTimestamp("17/07/16 16:37:45")); - final Resource containerAlloc = Resource.newInstance(1, 1); - jobMetaData.getResourceSkyline().setContainerSpec(containerAlloc); - jobMetaData.getResourceSkyline().setJobInputDataSize(1024.5); - jobMetaData.createSkyline(); - } - - @Test public final void testGetContainerSpec() { - final Resource containerAlloc = - jobMetaData.getResourceSkyline().getContainerSpec(); - final Resource containerAlloc2 = Resource.newInstance(1, 1); - Assert.assertEquals(containerAlloc.getMemorySize(), - containerAlloc2.getMemorySize()); - Assert.assertEquals(containerAlloc.getVirtualCores(), - containerAlloc2.getVirtualCores()); - } - - @Test public final void testGetJobSize() { - Assert.assertEquals(jobMetaData.getResourceSkyline().getJobInputDataSize(), - 1024.5, 0); - } - - @Test public final void testGetRecurrenceeId() { - final RecurrenceId recurrenceIdTest = - new RecurrenceId("Fraud Detection", "17/07/16 16:27:25"); - Assert.assertEquals(recurrenceIdTest, jobMetaData.getRecurrenceId()); - } - - @Test public final void testStringToUnixTimestamp() throws ParseException { - final long submissionTime = - logParserUtil.stringToUnixTimestamp("17/07/16 16:27:25"); - Assert.assertEquals(jobMetaData.getResourceSkyline().getJobSubmissionTime(), - submissionTime); - } - - @Test public final void testResourceSkyline() { - final RLESparseResourceAllocation skylineList = - jobMetaData.getResourceSkyline().getSkylineList(); - final int containerCPU = - jobMetaData.getResourceSkyline().getContainerSpec().getVirtualCores(); - int k; - for (k = 0; k < 5; k++) { - Assert.assertEquals(0, - skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); - } - for (k = 5; k < 15; k++) { - Assert.assertEquals(1, - skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); - } - for (k = 15; k < 605; k++) { - Assert.assertEquals(2, - skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); - } - for (k = 605; k < 615; k++) { - Assert.assertEquals(1, - skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); - } - Assert.assertEquals(0, - skylineList.getCapacityAtTime(615).getVirtualCores() / containerCPU); - } - - @Test public final void testContainerReleaseTimeMissing() - throws ParseException { - // create an invalid JobMetaData - recurrenceId = new RecurrenceId("Fraud Detection", "17/07/16 16:27:25"); - jobMetaData = new JobMetaData( - logParserUtil.stringToUnixTimestamp("17/07/16 16:27:25")); - jobMetaData.setRecurrenceId(recurrenceId); - jobMetaData.setContainerStart("C1", - logParserUtil.stringToUnixTimestamp("17/07/16 16:27:30")); - jobMetaData.setContainerEnd("C1", - logParserUtil.stringToUnixTimestamp("17/07/16 16:37:30")); - jobMetaData.setContainerStart("C2", - logParserUtil.stringToUnixTimestamp("17/07/16 16:27:40")); - jobMetaData.setJobFinishTime( - logParserUtil.stringToUnixTimestamp("17/07/16 16:37:45")); - final Resource containerAlloc = Resource.newInstance(1, 1); - jobMetaData.getResourceSkyline().setContainerSpec(containerAlloc); - jobMetaData.getResourceSkyline().setJobInputDataSize(1024.5); - jobMetaData.createSkyline(); - // test the generated ResourceSkyline - final RLESparseResourceAllocation skylineList = - jobMetaData.getResourceSkyline().getSkylineList(); - final int containerCPU = - jobMetaData.getResourceSkyline().getContainerSpec().getVirtualCores(); - int k; - for (k = 0; k < 5; k++) { - Assert.assertEquals(0, - skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); - } - for (k = 5; k < 605; k++) { - Assert.assertEquals(1, - skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); - } - Assert.assertEquals(0, - skylineList.getCapacityAtTime(605).getVirtualCores() / containerCPU); - } - - @After public final void cleanUp() { - jobMetaData = null; - recurrenceId = null; - logParserUtil = null; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestNativeParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestNativeParser.java b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestNativeParser.java deleted file mode 100644 index f399bf9..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestNativeParser.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.impl; - -import java.io.IOException; -import java.text.ParseException; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; -import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; -import org.apache.hadoop.resourceestimator.skylinestore.api.SkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore; -import org.apache.hadoop.resourceestimator.translator.api.LogParser; -import org.apache.hadoop.resourceestimator.translator.exceptions.DataFieldNotFoundException; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * This sample parser will parse the sample log and extract the resource - * skyline. - */ -public class TestNativeParser { - private LogParserUtil logParserUtil = new LogParserUtil(); - private SkylineStore skylineStore; - - @Before public final void setup() throws ResourceEstimatorException { - skylineStore = new InMemoryStore(); - final LogParser nativeParser = new BaseLogParser(); - Configuration config = new Configuration(); - config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE); - nativeParser.init(config, skylineStore); - logParserUtil.setLogParser(nativeParser); - } - - private void parseFile(final String logFile) - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - logParserUtil.parseLog(logFile); - } - - @Test public final void testParse() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/nativeLog.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = - new RecurrenceId("tpch_q12", "tpch_q12_0"); - final Map> jobSkylineLists = - skylineStore.getHistory(recurrenceId); - Assert.assertEquals(1, jobSkylineLists.size()); - final List jobHistory = jobSkylineLists.get(recurrenceId); - Assert.assertEquals(1, jobHistory.size()); - final ResourceSkyline resourceSkyline = jobHistory.get(0); - Assert.assertEquals(0, resourceSkyline.getJobInputDataSize(), 0); - Assert.assertEquals("tpch_q12_0", resourceSkyline.getJobId()); - Assert.assertEquals(0, resourceSkyline.getJobSubmissionTime()); - Assert.assertEquals(25, resourceSkyline.getJobFinishTime()); - Assert - .assertEquals(1024, resourceSkyline.getContainerSpec().getMemorySize()); - Assert - .assertEquals(1, resourceSkyline.getContainerSpec().getVirtualCores()); - final RLESparseResourceAllocation skylineLists = - resourceSkyline.getSkylineList(); - int k; - for (k = 0; k < 10; k++) { - Assert.assertEquals(1, - skylineLists.getCapacityAtTime(k).getMemorySize() / 1024); - } - for (k = 10; k < 15; k++) { - Assert.assertEquals(1074, - skylineLists.getCapacityAtTime(k).getMemorySize() / 1024); - } - for (k = 15; k < 20; k++) { - Assert.assertEquals(2538, - skylineLists.getCapacityAtTime(k).getMemorySize() / 1024); - } - for (k = 20; k < 25; k++) { - Assert.assertEquals(2468, - skylineLists.getCapacityAtTime(k).getMemorySize() / 1024); - } - Assert.assertEquals(0, - skylineLists.getCapacityAtTime(25).getMemorySize() / 1024); - } - - @After public final void cleanUp() { - skylineStore = null; - logParserUtil = null; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestRmParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestRmParser.java b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestRmParser.java deleted file mode 100644 index 4a39ed7..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestRmParser.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.impl; - -import java.io.IOException; -import java.text.ParseException; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; -import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; -import org.apache.hadoop.resourceestimator.skylinestore.api.SkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore; -import org.apache.hadoop.resourceestimator.translator.api.LogParser; -import org.apache.hadoop.resourceestimator.translator.exceptions.DataFieldNotFoundException; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * This sample parser will parse the sample log and extract the resource - * skyline. - */ -public class TestRmParser { - private LogParserUtil logParserUtil = new LogParserUtil(); - private SkylineStore skylineStore; - - @Before public final void setup() throws ResourceEstimatorException { - skylineStore = new InMemoryStore(); - final LogParser rmParser = new BaseLogParser(); - Configuration config = new Configuration(); - config.addResource(new org.apache.hadoop.fs.Path( - ResourceEstimatorConfiguration.CONFIG_FILE)); - config.set(ResourceEstimatorConfiguration.TRANSLATOR_LINE_PARSER, - RmSingleLineParser.class.getName()); - rmParser.init(config, skylineStore); - logParserUtil.setLogParser(rmParser); - } - - private void parseFile(final String logFile) - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - logParserUtil.parseLog(logFile); - } - - @Test public final void testParse() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/rmLog.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("FraudDetection", "1"); - final Map> jobSkylineLists = - skylineStore.getHistory(recurrenceId); - Assert.assertEquals(1, jobSkylineLists.size()); - final List jobHistory = jobSkylineLists.get(recurrenceId); - Assert.assertEquals(1, jobHistory.size()); - final ResourceSkyline resourceSkyline = jobHistory.get(0); - Assert.assertEquals(0, resourceSkyline.getJobInputDataSize(), 0); - Assert.assertEquals("application_1497832133857_0330", - resourceSkyline.getJobId()); - Assert.assertEquals( - logParserUtil.stringToUnixTimestamp("06/21/2017 16:10:13"), - resourceSkyline.getJobSubmissionTime()); - Assert.assertEquals( - logParserUtil.stringToUnixTimestamp("06/21/2017 16:18:35"), - resourceSkyline.getJobFinishTime()); - final Resource resource = Resource.newInstance(1800, 1); - Assert.assertEquals(resource.getMemorySize(), - resourceSkyline.getContainerSpec().getMemorySize()); - Assert.assertEquals(resource.getVirtualCores(), - resourceSkyline.getContainerSpec().getVirtualCores()); - final RLESparseResourceAllocation skylineLists = - resourceSkyline.getSkylineList(); - - int k; - for (k = 0; k < 142; k++) { - Assert.assertEquals(1, - skylineLists.getCapacityAtTime(k).getMemorySize() / resource - .getMemorySize()); - } - for (k = 142; k < 345; k++) { - Assert.assertEquals(2, - skylineLists.getCapacityAtTime(k).getMemorySize() / resource - .getMemorySize()); - } - for (k = 345; k < 502; k++) { - Assert.assertEquals(1, - skylineLists.getCapacityAtTime(k).getMemorySize() / resource - .getMemorySize()); - } - } - - @Test(expected = ParseException.class) - public final void testInvalidDateFormat() - throws ParseException { - logParserUtil.stringToUnixTimestamp("2017.07.16 16:37:45"); - } - - @Test public final void testDuplicateJobSubmissionTime() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog1.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "1"); - final Map> jobSkylineLists = - skylineStore.getHistory(recurrenceId); - Assert.assertEquals( - logParserUtil.stringToUnixTimestamp("06/21/2017 16:10:23"), - jobSkylineLists.get(recurrenceId).get(0).getJobSubmissionTime()); - } - - @Test public final void testJobIdNotFoundInJobSubmission() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog2.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "2"); - Assert.assertNull(skylineStore.getHistory(recurrenceId)); - } - - @Test public final void testJobIdNotFoundInContainerAlloc() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog3.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "3"); - final Map> jobSkylineLists = - skylineStore.getHistory(recurrenceId); - Assert.assertEquals(0, - jobSkylineLists.get(recurrenceId).get(0).getSkylineList() - .getCumulative().size()); - } - - @Test public final void testContainerIdNotFoundInContainerAlloc() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog4.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "4"); - final Map> jobSkylineLists = - skylineStore.getHistory(recurrenceId); - Assert.assertEquals(0, - jobSkylineLists.get(recurrenceId).get(0).getSkylineList() - .getCumulative().size()); - } - - @Test public final void testJobIdNotFoundInJobFailure() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog5.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "5"); - final Map> jobSkylineLists = - skylineStore.getHistory(recurrenceId); - Assert.assertEquals( - logParserUtil.stringToUnixTimestamp("06/21/2017 16:10:13"), - jobSkylineLists.get(recurrenceId).get(0).getJobSubmissionTime()); - } - - @Test public final void testJobIdNotFoundInJobFinish() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog6.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "6"); - Assert.assertNull(skylineStore.getHistory(recurrenceId)); - } - - @Test public final void testRecurrenceIdNotFoundInJobFinish() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog7.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "7"); - Assert.assertNull(skylineStore.getHistory(recurrenceId)); - } - - @Test public final void testJobIdNotFoundInResourceSpec() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog8.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "8"); - final Map> jobSkylineLists = - skylineStore.getHistory(recurrenceId); - Assert.assertEquals(1024, - jobSkylineLists.get(recurrenceId).get(0).getContainerSpec() - .getMemorySize()); - Assert.assertEquals(1, - jobSkylineLists.get(recurrenceId).get(0).getContainerSpec() - .getVirtualCores()); - } - - @Test public final void testResourceSpecNotFoundInResourceSpec() - throws SkylineStoreException, IOException, ParseException, - ResourceEstimatorException, DataFieldNotFoundException { - final String logFile = "src/test/resources/trace/invalidLog9.txt"; - parseFile(logFile); - final RecurrenceId recurrenceId = new RecurrenceId("Test", "9"); - final Map> jobSkylineLists = - skylineStore.getHistory(recurrenceId); - Assert.assertEquals(1024, - jobSkylineLists.get(recurrenceId).get(0).getContainerSpec() - .getMemorySize()); - Assert.assertEquals(1, - jobSkylineLists.get(recurrenceId).get(0).getContainerSpec() - .getVirtualCores()); - } - - @After public final void cleanUp() { - skylineStore = null; - logParserUtil = null; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/log4j.properties b/hadoop-tools/hadoop-resourceestimator/src/test/resources/log4j.properties deleted file mode 100644 index 60b8fbe..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/log4j.properties +++ /dev/null @@ -1,25 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -log4j.rootLogger=INFO, stdout -# Redirect log messages to console -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/answer.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/answer.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/answer.txt deleted file mode 100644 index 94565de..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/answer.txt +++ /dev/null @@ -1,7 +0,0 @@ -1 -1 -1738 -2535 -2484 -2478 -2468 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tinySample.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tinySample.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tinySample.txt deleted file mode 100644 index 15fb84e..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tinySample.txt +++ /dev/null @@ -1,2 +0,0 @@ -tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20| -tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0| 1:0|1:5|794:10|2517:15|2484:20| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tpch_q12.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tpch_q12.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tpch_q12.txt deleted file mode 100644 index ee19625..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tpch_q12.txt +++ /dev/null @@ -1,13 +0,0 @@ -tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0|8192:5|8798208:10|20791296:15|20348928:20|20275200:25|20217856:30|20160512:35|20045824:40|18554880:45|15736832:50|15302656:55|14991360:60|14737408:65|14508032:70|14360576:75|14196736:80|14049280:85|13901824:90|13795328:95|13721600:100|13664256:105|13565952:110|12877824:115|12148736:120|11476992:125|10829824:130|10444800:135|10059776:140|9633792:145|9355264:150|9125888:155|9027584:160|8937472:165|8839168:170|8757248:175|8642560:180|8527872:185|8421376:190|8396800:195|8355840:200|8323072:205|8298496:210|8282112:215|8282112:220|8273920:225|8273920:230|8273920:235|8273920:240|8273920:245|8273920:250|8273920:255|7700480:260|5160960:265|4087808:270|3907584:275|3702784:280|3407872:285|2965504:290|2686976:295|2473984:300|2244608:305|2056192:310|1933312:315|1810432:320|1613824:325|1417216:330|1277952:335|1130496:340|843776:345|679936:350|647168:355|630784:360|630784:365|630784:370|614400:375|581632:380|548864:385|540672: 390|540672:395|540672:400|507904:405|491520:410|278528:415|16384:420|0:425| 1:0|1:5|1074:10|2538:15|2484:20|2475:25|2468:30| -tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0|8192:5|6504448:10|20619264:15|20348928:20|20299776:25|20201472:30|20127744:35|20021248:40|18366464:45|15392768:50|15605760:55|15400960:60|16064512:65|13664256:70|15753216:75|15843328:80|15843328:85|13615104:90|11943936:95|11501568:100|10895360:105|10223616:110|9822208:115|9478144:120|9142272:125|8855552:130|8683520:135|8544256:140|8413184:145|8331264:150|8290304:155|8282112:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|8273920:190|8265728:195|4890624:200|1089536:205|671744:210|565248:215|516096:220|483328:225|360448:230|16384:235|16384:240|8192:245|0:250| 1:0|1:5|794:10|2517:15|2484:20|2478:25|2466:30| -tpch_q12 50 1462150611000 1462150620642 1462150895642 tpch_q12_7 8192:0|8192:5|11362304:10|20365312:15|20348928:20|20242432:25|20176896:30|20029440:35|19783680:40|16703488:45|15286272:50|14589952:55|14172160:60|13770752:65|13598720:70|12992512:75|12468224:80|12025856:85|11640832:90|11460608:95|11272192:100|10510336:105|9789440:110|9207808:115|8781824:120|8585216:125|8478720:130|8396800:135|8314880:140|8290304:145|8273920:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|7077888:185|4407296:190|3735552:195|3260416:200|2736128:205|2228224:210|1777664:215|1433600:220|1155072:225|983040:230|843776:235|720896:240|647168:245|630784:250|622592:255|565248:260|303104:265|16384:270|8192:275|0:280| 1:0|1:5|1387:10|2486:15|2484:20|2471:25|2463:30| -tpch_q12 50 1462150893000 1462150904200 1462151134200 tpch_q12_8 8192:0|8192:5|7946240:10|20389888:15|20348928:20|20267008:25|20160512:30|20054016:35|19824640:40|17866752:45|14041088:50|14114816:55|12615680:60|14336000:65|12771328:70|12189696:75|11796480:80|11599872:85|11517952:90|11460608:95|11403264:100|11116544:105|10010624:110|9306112:115|8847360:120|8593408:125|8404992:130|8331264:135|8298496:140|8290304:145|8282112:150|8282112:155|8282112:160|8282112:165|8282112:170|8282112:175|8273920:180|8273920:185|8273920:190|8273920:195|8273920:200|8273920:205|8118272:210|2883584:215|376832:220|16384:225|8192:230|0:235| 1:0|1:5|970:10|2489:15|2484:20|2474:25|2461:30| -tpch_q12 50 1462151132000 1462151141241 1462151421241 tpch_q12_9 8192:0|16384:5|16031744:10|20348928:15|20348928:20|20201472:25|20152320:30|20004864:35|19562496:40|16121856:45|15319040:50|14589952:55|14139392:60|13770752:65|13131776:70|12378112:75|11878400:80|11649024:85|11534336:90|11444224:95|11157504:100|10362880:105|9650176:110|9134080:115|8749056:120|8519680:125|8372224:130|8306688:135|8290304:140|8282112:145|8282112:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|7569408:190|4505600:195|3522560:200|3072000:205|2801664:210|2523136:215|2080768:220|1679360:225|1343488:230|1007616:235|892928:240|819200:245|688128:250|573440:255|540672:260|524288:265|491520:270|24576:275|8192:280|0:285| 1:0|2:5|1957:10|2484:15|2484:20|2466:25|2460:30| -tpch_q12 50 1462151420000 1462151429196 1462151739196 tpch_q12_10 8192:0|8192:5|10543104:10|20357120:15|20348928:20|20234240:25|20168704:30|20045824:35|19750912:40|17293312:45|15327232:50|14770176:55|14336000:60|13926400:65|13410304:70|12517376:75|12034048:80|11714560:85|11526144:90|11427840:95|11247616:100|10616832:105|9928704:110|9412608:115|8962048:120|8585216:125|8380416:130|8323072:135|8290304:140|8282112:145|8282112:150|8282112:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|8273920:190|6889472:195|4120576:200|3465216:205|3153920:210|2924544:215|2514944:220|2056192:225|1794048:230|1646592:235|1417216:240|1122304:245|1048576:250|1007616:255|892928:260|778240:265|647168:270|557056:275|532480:280|524288:285|507904:290|475136:295|229376:300|16384:305|8192:310|0:315| 1:0|1:5|1287:10|2485:15|2484:20|2470:25|2462:30| -tpch_q12 50 1462151735000 1462151744905 1462152069905 tpch_q12_11 8192:0|8192:5|9494528:10|20389888:15|20348928:20|20267008:25|20168704:30|20086784:35|19652608:40|17334272:45|15351808:50|15040512:55|14639104:60|14295040:65|13967360:70|13672448:75|12853248:80|12181504:85|11730944:90|11493376:95|11255808:100|10592256:105|10002432:110|9633792:115|9224192:120|8896512:125|8634368:130|8462336:135|8355840:140|8298496:145|8282112:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8142848:185|5718016:190|4440064:195|3989504:200|3604480:205|3334144:210|2981888:215|2572288:220|2269184:225|1966080:230|1744896:235|1507328:240|1310720:245|1089536:250|942080:255|851968:260|794624:265|745472:270|655360:275|598016:280|565248:285|548864:290|532480:295|532480:300|532480:305|491520:310|270336:315|16384:320|8192:325|0:330| 1:0|1:5|1159:10|2489:15|2484:20|2474:25|2462:30| -tpch_q12 50 1462152067000 1462152075862 1462152365862 tpch_q12_12 8192:0|8192:5|11583488:10|20357120:15|20348928:20|20234240:25|20168704:30|20037632:35|19734528:40|16744448:45|15351808:50|14704640:55|14229504:60|13901824:65|13631488:70|13107200:75|12500992:80|12140544:85|11804672:90|11558912:95|11386880:100|10723328:105|9928704:110|9355264:115|8937472:120|8691712:125|8519680:130|8421376:135|8372224:140|8339456:145|8306688:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|7577600:190|4898816:195|3719168:200|3104768:205|2613248:210|2252800:215|2064384:220|1835008:225|1515520:230|1171456:235|974848:240|892928:245|802816:250|729088:255|712704:260|663552:265|557056:270|483328:275|368640:280|16384:285|8192:290|0:295| 1:0|1:5|1414:10|2485:15|2484:20|2470:25|2462:30| -tpch_q12 50 1462161466000 1462161474815 1462161689815 tpch_q12_45 8192:0|8192:5|13836288:10|20365312:15|20348928:20|20209664:25|20160512:30|20029440:35|19906560:40|18333696:45|15794176:50|16015360:55|13451264:60|15269888:65|12730368:70|11862016:75|11657216:80|11509760:85|11460608:90|11403264:95|11296768:100|10698752:105|9814016:110|9175040:115|8757248:120|8486912:125|8339456:130|8290304:135|8282112:140|8273920:145|8273920:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|5447680:180|1245184:185|737280:190|589824:195|483328:200|147456:205|16384:210|8192:215|0:220| 1:0|1:5|1689:10|2486:15|2484:20|2467:25|2461:30| -tpch_q12 50 1462161685000 1462161693550 1462161903550 tpch_q12_46 8192:0|8192:5|13500416:10|20348928:15|20348928:20|20201472:25|20152320:30|20037632:35|19734528:40|17047552:45|14548992:50|13778944:55|14696448:60|16064512:65|14540800:70|12468224:75|11935744:80|11640832:85|11517952:90|11436032:95|11190272:100|10379264:105|9740288:110|9199616:115|8863744:120|8585216:125|8429568:130|8331264:135|8290304:140|8282112:145|8282112:150|8282112:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|8257536:190|3710976:195|393216:200|16384:205|8192:210|0:215| 1:0|1:5|1648:10|2484:15|2484:20|2466:25|2460:30| -tpch_q12 50 1462161901000 1462161909461 1462162219461 tpch_q12_47 8192:0|8192:5|14802944:10|20348928:15|20348928:20|20217856:25|20152320:30|20013056:35|19734528:40|16285696:45|15310848:50|14573568:55|14090240:60|13639680:65|13025280:70|12271616:75|11911168:80|11673600:85|11501568:90|11411456:95|11206656:100|10493952:105|9797632:110|9175040:115|8749056:120|8503296:125|8396800:130|8347648:135|8331264:140|8290304:145|8273920:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|6823936:185|4120576:190|3342336:195|2711552:200|2170880:205|1712128:210|1466368:215|1294336:220|1187840:225|1089536:230|917504:235|770048:240|663552:245|589824:250|548864:255|540672:260|540672:265|540672:270|540672:275|540672:280|540672:285|540672:290|516096:295|409600:300|16384:305|8192:310|0:315| 1:0|1:5|1807:10|2484:15|2484:20|2468:25|2460:30| -tpch_q12 50 1462162218000 1462162226670 1462162536670 tpch_q12_48 8192:0|8192:5|13295616:10|20348928:15|20348928:20|20234240:25|20160512:30|20070400:35|19898368:40|15876096:45|15138816:50|14344192:55|14024704:60|13787136:65|13623296:70|13312000:75|12607488:80|12042240:85|11780096:90|11640832:95|11493376:100|10649600:105|9732096:110|9232384:115|8953856:120|8749056:125|8577024:130|8429568:135|8347648:140|8298496:145|8290304:150|8290304:155|8290304:160|8282112:165|8282112:170|8282112:175|8282112:180|8273920:185|8273920:190|8273920:195|8273920:200|8273920:205|8273920:210|8273920:215|7798784:220|7307264:225|4808704:230|3891200:235|3530752:240|3162112:245|2596864:250|2195456:255|1835008:260|1589248:265|1425408:270|1204224:275|974848:280|745472:285|581632:290|483328:295|237568:300|16384:305|8192:310|0:315| 1:0|1:5|1623:10|2484:15|2484:20|2470:25|2461:30| -tpch_q12 50 1462162533000 1462162541572 1462162766572 tpch_q12_49 8192:0|8192:5|13320192:10|20348928:15|20348928:20|20217856:25|20152320:30|20004864:35|19750912:40|18366464:45|18063360:50|17530880:55|12361728:60|14123008:65|12713984:70|12181504:75|11943936:80|11730944:85|11624448:90|11493376:95|11329536:100|10723328:105|9887744:110|9322496:115|8921088:120|8634368:125|8429568:130|8339456:135|8314880:140|8306688:145|8298496:150|8282112:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|8232960:190|3375104:195|696320:200|581632:205|507904:210|335872:215|16384:220|8192:225|0:230| 1:0|1:5|1626:10|2484:15|2484:20|2468:25|2460:30| http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceEstimatorService.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceEstimatorService.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceEstimatorService.txt deleted file mode 100644 index 15fb84e..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceEstimatorService.txt +++ /dev/null @@ -1,2 +0,0 @@ -tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20| -tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0| 1:0|1:5|794:10|2517:15|2484:20| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceestimator-config.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceestimator-config.xml b/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceestimator-config.xml deleted file mode 100644 index f78085a..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceestimator-config.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - resourceestimator.solver.lp.alpha - 0.1 - - The resource estimator has an integrated Linear Programming solver to make the prediction, and this parameter tunes the tradeoff between resource over-allocation and under-allocation in the Linear Programming model. This parameter varies from 0 to 1, and a larger alpha value means the model minimizes over-allocation better. Default value is 0.1. - - - - resourceestimator.solver.lp.beta - 0.1 - - This parameter controls the generalization of the Linear Programming model. This parameter varies from 0 to 1. Deafult value is 0.1. - - - - resourceestimator.solver.lp.minJobRuns - 2 - - The minimum number of job runs required in order to make the prediction. Default value is 2. - - - - resourceestimator.timeInterval - 5 - - The time length which is used to discretize job execution into intervals. Note that the estimator makes resource allocation prediction for each interval. A smaller time interval has more fine-grained granularity for prediction, but it also takes longer time and more space for prediction. Default value is 5 (seconds). - - - - resourceestimator.translator.line-parser - org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser - - The class name of the translator single-line parser, which parses a single line in the log. Default value is org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser, which can parse one line in the sample log. Note that if users want to parse Hadoop Resource Manager logs, they need to set the value to be org.apache.hadoop.resourceestimator.translator.impl.RmSingleLineParser. If they want to implement single-line parser to parse their customized log file, they need to change this value accordingly. - - - - - http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog1.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog1.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog1.txt deleted file mode 100644 index ba05a89..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog1.txt +++ /dev/null @@ -1,4 +0,0 @@ -// Test case 1: duplicate job submission time -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_1 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252823:e,06/21/2017 16:10:23,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_1 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_1,name=Test-1,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog2.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog2.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog2.txt deleted file mode 100644 index fe24943..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog2.txt +++ /dev/null @@ -1,3 +0,0 @@ -// Test case 2: no jobId found in job submission log -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPIDs=2 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_2,name=Test-2,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog3.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog3.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog3.txt deleted file mode 100644 index 63d9c2c..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog3.txt +++ /dev/null @@ -1,5 +0,0 @@ -// Test case 3: no jobId found in container allocation log -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_3 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPIDs=3 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_3 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_3,name=Test-3,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog4.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog4.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog4.txt deleted file mode 100644 index d787319..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog4.txt +++ /dev/null @@ -1,5 +0,0 @@ -// Test case 4: no containerId found in container allocation log -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_4 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_4 CONTAINERIDs=container_e26_1497832133857_0330_01_000001" -253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_4 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_4,name=Test-4,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog5.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog5.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog5.txt deleted file mode 100644 index 2bafa67..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog5.txt +++ /dev/null @@ -1,7 +0,0 @@ -// Test case 5: jobId not found in app failure log -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_5 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_5 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_5 CONTAINERID=container_e26_1497832133857_0330_01_000001" -e,06/20/2017 22:30:21,yarn resourcemanager,DefaultTag,Pid="5156" Tid="9896" TS="0x01D2EA14CDBCF8F1" String1="17/06/20 22:30:21 INFO capacity.CapacityScheduler: Application Attempt appattempt_5 is done. finalState=FAILED" -252844:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO attempt.RMAppAttemptImpl: Storing attempt: AppId: application_5 AttemptId: appattempt_5_000001 MasterContainer: Container: [ContainerId: container_e26_1497832133857_0330_01_000001, NodeId: by2pr08mb1799.namprd08.prod.outlook.com:10131, NodeHttpAddress: by2pr08mb1799.namprd08.prod.outlook.com:8042, Resource: , Priority: 0, Token: Token { kind: ContainerToken, service: by2pr08mb1799.namprd08.prod.outlook.com:10131 }, ]" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_5,name=Test-5,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog6.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog6.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog6.txt deleted file mode 100644 index bcb5112..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog6.txt +++ /dev/null @@ -1,5 +0,0 @@ -// Test case 6: no jobId found in job finish log -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_6 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_6 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_6 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appIds=application_6,name=Test-6,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog7.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog7.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog7.txt deleted file mode 100644 index 46676be..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog7.txt +++ /dev/null @@ -1,5 +0,0 @@ -// Test case 7: no recurrenceId found in job finish log -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_7 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_7 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_7 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_7,name=Test:7,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog8.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog8.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog8.txt deleted file mode 100644 index b8a8e20..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog8.txt +++ /dev/null @@ -1,6 +0,0 @@ -// Test case 8: no jobId found in resource spec log -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_8 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_8 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_8 CONTAINERID=container_e26_1497832133857_0330_01_000001" -252844:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO attempt.RMAppAttemptImpl: Storing attempt: AppId: application_8 AttemptIds: appattempt_8_000001 MasterContainer: Container: [ContainerId: container_e26_1497832133857_0330_01_000001, NodeId: by2pr08mb1799.namprd08.prod.outlook.com:10131, NodeHttpAddress: by2pr08mb1799.namprd08.prod.outlook.com:8042, Resource: , Priority: 0, Token: Token { kind: ContainerToken, service: by2pr08mb1799.namprd08.prod.outlook.com:10131 }, ]" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_8,name=Test-8,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog9.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog9.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog9.txt deleted file mode 100644 index 598f935..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog9.txt +++ /dev/null @@ -1,6 +0,0 @@ -// Test case 9: no resource spec found in resource spec log -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_9 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_9 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_9 CONTAINERID=container_e26_1497832133857_0330_01_000001" -252844:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO attempt.RMAppAttemptImpl: Storing attempt: AppId: application_9 AttemptId: appattempt_9_000001 MasterContainer: Container: [ContainerId: container_e26_1497832133857_0330_01_000001, NodeId: by2pr08mb1799.namprd08.prod.outlook.com:10131, NodeHttpAddress: by2pr08mb1799.namprd08.prod.outlook.com:8042, Resource: , Priority: 0, Token: Token { kind: ContainerToken, service: by2pr08mb1799.namprd08.prod.outlook.com:10131 }, ]" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_9,name=Test-9,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/nativeLog.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/nativeLog.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/nativeLog.txt deleted file mode 100644 index fd03ea3..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/nativeLog.txt +++ /dev/null @@ -1 +0,0 @@ -tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/rmLog.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/rmLog.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/rmLog.txt deleted file mode 100644 index 7532a73..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/rmLog.txt +++ /dev/null @@ -1,30 +0,0 @@ -252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_1497832133857_0330 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" -252825:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO rmapp.RMAppImpl: Storing application with id application_1497832133857_0330" -252826:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from NEW to NEW_SAVING" -252827:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO recovery.RMStateStore: Storing info for app: application_1497832133857_0330" -252828:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from NEW_SAVING to SUBMITTED" -252829:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO capacity.ParentQueue: Application added - appId: application_1497832133857_0330 user: hadoop leaf-queue of parent: root #applications: 3" -252830:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO capacity.CapacityScheduler: Accepted application application_1497832133857_0330 from user: hadoop, in queue: PROD" -252831:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from SUBMITTED to ACCEPTED" -252834:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO capacity.LeafQueue: Application application_1497832133857_0330 from user: hadoop activated in queue: PROD" -252835:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO capacity.LeafQueue: Application added - appId: application_1497832133857_0330 user: org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue$User@7e2d171f, leaf-queue: PROD #user-pending-applications: 0 #user-active-applications: 1 #queue-pending-applications: 0 #queue-active-applications: 1" -252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1497832133857_0330 CONTAINERID=container_e26_1497832133857_0330_01_000001" -252844:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO attempt.RMAppAttemptImpl: Storing attempt: AppId: application_1497832133857_0330 AttemptId: appattempt_1497832133857_0330_000001 MasterContainer: Container: [ContainerId: container_e26_1497832133857_0330_01_000001, NodeId: by2pr08mb1799.namprd08.prod.outlook.com:10131, NodeHttpAddress: by2pr08mb1799.namprd08.prod.outlook.com:8042, Resource: , Priority: 0, Token: Token { kind: ContainerToken, service: by2pr08mb1799.namprd08.prod.outlook.com:10131 }, ]" -252897:e,06/21/2017 16:12:34,yarn resourcemanager,DefaultTag,Pid="5156" Tid="41712" TS="0x01D2EAA931905154" String1="17/06/21 16:12:34 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.25 OPERATION=Register App Master TARGET=ApplicationMasterService RESULT=SUCCESS APPID=application_1497832133857_0330 APPATTEMPTID=appattempt_1497832133857_0330_000001" -252900:e,06/21/2017 16:12:34,yarn resourcemanager,DefaultTag,Pid="5156" Tid="41712" TS="0x01D2EAA931905154" String1="17/06/21 16:12:34 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from ACCEPTED to RUNNING" -252906:e,06/21/2017 16:12:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="41712" TS="0x01D2EAA93234D4E8" String1="17/06/21 16:12:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1497832133857_0330 CONTAINERID=container_e26_1497832133857_0330_01_000002" -252910:e,06/21/2017 16:12:38,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA933A6643E" String1="17/06/21 16:12:38 INFO scheduler.AppSchedulingInfo: checking for deactivate of application :application_1497832133857_0330" -253016:e,06/21/2017 16:15:58,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39188" TS="0x01D2EAA9AB29C5EE" String1="17/06/21 16:15:58 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1497832133857_0330 CONTAINERID=container_e26_1497832133857_0330_01_000002" -253086:e,06/21/2017 16:18:25,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA02A6A418" String1="17/06/21 16:18:25 INFO rmapp.RMAppImpl: Updating application application_1497832133857_0330 with final state: FINISHING" -253087:e,06/21/2017 16:18:25,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA02A6A418" String1="17/06/21 16:18:25 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from RUNNING to FINAL_SAVING" -253089:e,06/21/2017 16:18:25,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA02B28FF0" String1="17/06/21 16:18:25 INFO recovery.RMStateStore: Updating info for app: application_1497832133857_0330" -253090:e,06/21/2017 16:18:25,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA02B28FF0" String1="17/06/21 16:18:25 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from FINAL_SAVING to FINISHING" -253091:e,06/21/2017 16:18:26,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA033F3BC8" String1="17/06/21 16:18:26 INFO resourcemanager.ApplicationMasterService: application_1497832133857_0330 unregistered successfully. " -253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1497832133857_0330 CONTAINERID=container_e26_1497832133857_0330_01_000001" -253105:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from FINISHING to FINISHED" -253110:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO scheduler.AppSchedulingInfo: Application application_1497832133857_0330 requests cleared" -253111:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO capacity.LeafQueue: Application removed - appId: application_1497832133857_0330 user: hadoop queue: PROD #user-pending-applications: 0 #user-active-applications: 0 #queue-pending-applications: 0 #queue-active-applications: 0" -253112:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO capacity.ParentQueue: Application removed - appId: application_1497832133857_0330 user: hadoop leaf-queue of parent: root #applications: 2" -253113:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=Application Finished - Succeeded TARGET=RMAppManager RESULT=SUCCESS APPID=application_1497832133857_0330" -253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_1497832133857_0330,name=FraudDetection-1,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/application_1497832133857_0330/,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=,applicationType=MAPREDUCE" -e,06/20/2017 22:30:21,yarn resourcemanager,DefaultTag,Pid="5156" Tid="9896" TS="0x01D2EA14CDBCF8F1" String1="17/06/20 22:30:21 INFO capacity.CapacityScheduler: Application Attempt appattempt_1497832133857_0194_000001 is done. finalState=FAILED" http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-tools-dist/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 7d57843..61fdddb 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -112,12 +112,6 @@ compile - org.apache.hadoop - hadoop-resourceestimator - ${project.version} - compile - - org.apache.hadoop hadoop-azure-datalake compile http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index db67639..86d573a 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -44,7 +44,6 @@ hadoop-pipes hadoop-openstack hadoop-sls - hadoop-resourceestimator hadoop-aws hadoop-azure hadoop-azure-datalake --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org