Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 42DE5F3B1 for ; Tue, 26 Mar 2013 18:19:12 +0000 (UTC) Received: (qmail 24184 invoked by uid 500); 26 Mar 2013 18:19:12 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 24111 invoked by uid 500); 26 Mar 2013 18:19:11 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 24103 invoked by uid 99); 26 Mar 2013 18:19:11 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Mar 2013 18:19:11 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Mar 2013 18:19:08 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 1734B23889BB for ; Tue, 26 Mar 2013 18:18:46 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1461254 - in /hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src: main/java/org/apache/hadoop/mapred/gridmix/ main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ test/java/org/apache/hadoop/mapred/gridmix/ Date: Tue, 26 Mar 2013 18:18:45 -0000 To: common-commits@hadoop.apache.org From: sseth@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130326181846.1734B23889BB@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: sseth Date: Tue Mar 26 18:18:44 2013 New Revision: 1461254 URL: http://svn.apache.org/r1461254 Log: MAPREDUCE-5077. Remove mapreduce.util.ResourceCalculatorPlugin and related code. Contributed by Karthik Kambatla. Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1461254&r1=1461253&r2=1461254&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java Tue Mar 26 18:18:44 2013 @@ -36,11 +36,11 @@ import org.apache.hadoop.mapreduce.TaskI import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.tools.rumen.JobStory; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; import org.apache.hadoop.tools.rumen.TaskInfo; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import java.io.IOException; import java.security.PrivilegedExceptionAction; Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java?rev=1461254&r1=1461253&r2=1461254&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java Tue Mar 26 18:18:44 2013 @@ -22,8 +22,8 @@ import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.gridmix.Progressive; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** *

A {@link ResourceUsageEmulatorPlugin} that emulates the cumulative CPU @@ -166,7 +166,7 @@ implements ResourceUsageEmulatorPlugin { */ public void calibrate(ResourceCalculatorPlugin monitor, long totalCpuUsage) { - long initTime = monitor.getProcResourceValues().getCumulativeCpuTime(); + long initTime = monitor.getCumulativeCpuTime(); long defaultLoopSize = 0; long finalTime = initTime; @@ -175,7 +175,7 @@ implements ResourceUsageEmulatorPlugin { while (finalTime - initTime < 100) { // 100 ms ++defaultLoopSize; performUnitComputation(); //perform unit computation - finalTime = monitor.getProcResourceValues().getCumulativeCpuTime(); + finalTime = monitor.getCumulativeCpuTime(); } long referenceRuntime = finalTime - initTime; @@ -250,7 +250,7 @@ implements ResourceUsageEmulatorPlugin { // section long currentCpuUsage = - monitor.getProcResourceValues().getCumulativeCpuTime(); + monitor.getCumulativeCpuTime(); // estimate the cpu usage rate float rate = (currentCpuUsage - lastSeenCpuUsageCpuUsage) / (currentProgress - lastSeenProgress); @@ -264,7 +264,7 @@ implements ResourceUsageEmulatorPlugin { (long)(targetCpuUsage * getWeightForProgressInterval(currentProgress)); - while (monitor.getProcResourceValues().getCumulativeCpuTime() + while (monitor.getCumulativeCpuTime() < currentWeighedTarget) { emulatorCore.compute(); // sleep for 100ms @@ -282,7 +282,7 @@ implements ResourceUsageEmulatorPlugin { lastSeenProgress = progress.getProgress(); // set the last seen usage lastSeenCpuUsageCpuUsage = - monitor.getProcResourceValues().getCumulativeCpuTime(); + monitor.getCumulativeCpuTime(); } } } Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java?rev=1461254&r1=1461253&r2=1461254&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java Tue Mar 26 18:18:44 2013 @@ -20,7 +20,7 @@ package org.apache.hadoop.mapred.gridmix import java.io.IOException; import org.apache.hadoop.mapred.gridmix.Progressive; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; import org.apache.hadoop.conf.Configuration; Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java?rev=1461254&r1=1461253&r2=1461254&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java Tue Mar 26 18:18:44 2013 @@ -22,9 +22,9 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.gridmix.Progressive; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** *

This is the driver class for managing all the resource usage emulators. Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java?rev=1461254&r1=1461253&r2=1461254&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java Tue Mar 26 18:18:44 2013 @@ -21,8 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.gridmix.Progressive; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** *

A {@link ResourceUsageEmulatorPlugin} that emulates the total heap Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java?rev=1461254&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java Tue Mar 26 18:18:44 2013 @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.gridmix; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; + +/** + * Plugin class to test resource information reported by NM. Use configuration + * items {@link #MAXVMEM_TESTING_PROPERTY} and {@link #MAXPMEM_TESTING_PROPERTY} + * to tell NM the total vmem and the total pmem. Use configuration items + * {@link #NUM_PROCESSORS}, {@link #CPU_FREQUENCY}, {@link #CUMULATIVE_CPU_TIME} + * and {@link #CPU_USAGE} to tell TT the CPU information. + */ +@InterfaceAudience.Private +public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin { + + /** max vmem on the TT */ + public static final String MAXVMEM_TESTING_PROPERTY = + "mapred.tasktracker.maxvmem.testing"; + /** max pmem on the TT */ + public static final String MAXPMEM_TESTING_PROPERTY = + "mapred.tasktracker.maxpmem.testing"; + /** number of processors for testing */ + public static final String NUM_PROCESSORS = + "mapred.tasktracker.numprocessors.testing"; + /** CPU frequency for testing */ + public static final String CPU_FREQUENCY = + "mapred.tasktracker.cpufrequency.testing"; + /** cumulative CPU usage time for testing */ + public static final String CUMULATIVE_CPU_TIME = + "mapred.tasktracker.cumulativecputime.testing"; + /** CPU usage percentage for testing */ + public static final String CPU_USAGE = "mapred.tasktracker.cpuusage.testing"; + /** process cumulative CPU usage time for testing */ + public static final String PROC_CUMULATIVE_CPU_TIME = + "mapred.tasktracker.proccumulativecputime.testing"; + /** process pmem for testing */ + public static final String PROC_PMEM_TESTING_PROPERTY = + "mapred.tasktracker.procpmem.testing"; + /** process vmem for testing */ + public static final String PROC_VMEM_TESTING_PROPERTY = + "mapred.tasktracker.procvmem.testing"; + + /** {@inheritDoc} */ + @Override + public long getVirtualMemorySize() { + return getConf().getLong(MAXVMEM_TESTING_PROPERTY, -1); + } + + /** {@inheritDoc} */ + @Override + public long getPhysicalMemorySize() { + return getConf().getLong(MAXPMEM_TESTING_PROPERTY, -1); + } + + /** {@inheritDoc} */ + @Override + public long getAvailableVirtualMemorySize() { + return getConf().getLong(MAXVMEM_TESTING_PROPERTY, -1); + } + + /** {@inheritDoc} */ + @Override + public long getAvailablePhysicalMemorySize() { + return getConf().getLong(MAXPMEM_TESTING_PROPERTY, -1); + } + + /** {@inheritDoc} */ + @Override + public int getNumProcessors() { + return getConf().getInt(NUM_PROCESSORS, -1); + } + + /** {@inheritDoc} */ + @Override + public long getCpuFrequency() { + return getConf().getLong(CPU_FREQUENCY, -1); + } + + /** {@inheritDoc} */ + @Override + public long getCumulativeCpuTime() { + return getConf().getLong(CUMULATIVE_CPU_TIME, -1); + } + + /** {@inheritDoc} */ + @Override + public float getCpuUsage() { + return getConf().getFloat(CPU_USAGE, -1); + } +} Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java?rev=1461254&r1=1461253&r2=1461254&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java Tue Mar 26 18:18:44 2013 @@ -23,7 +23,6 @@ import static org.junit.Assert.*; import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.gridmix.DebugJobProducer.MockJob; import org.apache.hadoop.mapred.gridmix.TestHighRamJob.DummyGridmixJob; @@ -32,8 +31,8 @@ import org.apache.hadoop.mapred.gridmix. import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.TotalHeapUsageEmulatorPlugin.DefaultHeapUsageEmulator; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** * Test Gridmix memory emulation. Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java?rev=1461254&r1=1461253&r2=1461254&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java Tue Mar 26 18:18:44 2013 @@ -31,14 +31,13 @@ import org.apache.hadoop.mapreduce.TaskI import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; import org.apache.hadoop.mapreduce.task.MapContextImpl; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; -import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin; import org.apache.hadoop.mapred.gridmix.LoadJob.ResourceUsageMatcherRunner; import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin; import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.ResourceUsageEmulatorPlugin; import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.ResourceUsageMatcher; import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin.DefaultCpuUsageEmulator; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** * Test Gridmix's resource emulator framework and supported plugins. @@ -234,16 +233,6 @@ public class TestResourceUsageEmulators public long getCumulativeCpuTime() { return core.getCpuUsage(); } - - /** - * Returns a {@link ProcResourceValues} with cumulative cpu usage - * computed using {@link #getCumulativeCpuTime()}. - */ - @Override - public ProcResourceValues getProcResourceValues() { - long usageValue = getCumulativeCpuTime(); - return new ProcResourceValues(usageValue, -1, -1); - } } /**