Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8BE04200C65 for ; Sat, 15 Apr 2017 02:36:38 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 8A838160BA3; Sat, 15 Apr 2017 00:36:38 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id AB8C7160B8C for ; Sat, 15 Apr 2017 02:36:37 +0200 (CEST) Received: (qmail 89467 invoked by uid 500); 15 Apr 2017 00:36:36 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 89458 invoked by uid 99); 15 Apr 2017 00:36:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 15 Apr 2017 00:36:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9C739DFF0F; Sat, 15 Apr 2017 00:36:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: haibochen@apache.org To: common-commits@hadoop.apache.org Message-Id: <88b4ddf329154a8792f57d66dfd3ecbe@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: MAPREDUCE-6673. Add a test example job that grows in memory usage over time (Karthik Kambatla via Haibo Chen) Date: Sat, 15 Apr 2017 00:36:36 +0000 (UTC) archived-at: Sat, 15 Apr 2017 00:36:38 -0000 Repository: hadoop Updated Branches: refs/heads/trunk 0ac17dc64 -> 25ac44709 MAPREDUCE-6673. Add a test example job that grows in memory usage over time (Karthik Kambatla via Haibo Chen) Change-Id: Iccfc8c67c38c526cc61726d87bfcbcf69ac36fea Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25ac4470 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25ac4470 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25ac4470 Branch: refs/heads/trunk Commit: 25ac44709b4bbed78b607ea48021237b64e01b9f Parents: 0ac17dc Author: Haibo Chen Authored: Fri Apr 14 17:33:04 2017 -0700 Committer: Haibo Chen Committed: Fri Apr 14 17:36:03 2017 -0700 ---------------------------------------------------------------------- .../hadoop/mapreduce/GrowingSleepJob.java | 68 ++++++++++++++++++++ .../apache/hadoop/test/MapredTestDriver.java | 3 + 2 files changed, 71 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/25ac4470/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java new file mode 100644 index 0000000..55740f7 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.util.ToolRunner; + +import java.io.IOException; +import java.util.ArrayList; + +/** + * A sleep job whose mappers create 1MB buffer for every record. + */ +public class GrowingSleepJob extends SleepJob { + private static final Log LOG = LogFactory.getLog(GrowingSleepJob.class); + + public static class GrowingSleepMapper extends SleepMapper { + private final int MB = 1024 * 1024; + private ArrayList bytes = new ArrayList<>(); + + @Override + public void map(IntWritable key, IntWritable value, Context context) + throws IOException, InterruptedException { + super.map(key, value, context); + long free = Runtime.getRuntime().freeMemory(); + if (free > 32 * MB) { + LOG.info("Free memory = " + free + + " bytes. Creating 1 MB on the heap."); + bytes.add(new byte[MB]); + } + } + } + + public static void main(String[] args) throws Exception { + int res = ToolRunner.run(new Configuration(), new GrowingSleepJob(), args); + System.exit(res); + } + + @Override + public Job createJob(int numMapper, int numReducer, + long mapSleepTime, int mapSleepCount, + long reduceSleepTime, int reduceSleepCount) + throws IOException { + Job job = super.createJob(numMapper, numReducer, mapSleepTime, + mapSleepCount, reduceSleepTime, reduceSleepCount); + job.setMapperClass(GrowingSleepMapper.class); + job.setJobName("Growing sleep job"); + return job; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/25ac4470/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java index 76198b8..a2a13d5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java @@ -42,6 +42,7 @@ import org.apache.hadoop.mapred.TestSequenceFileInputFormat; import org.apache.hadoop.mapred.TestTextInputFormat; import org.apache.hadoop.mapred.ThreadedMapBenchmark; import org.apache.hadoop.mapreduce.FailJob; +import org.apache.hadoop.mapreduce.GrowingSleepJob; import org.apache.hadoop.mapreduce.LargeSorter; import org.apache.hadoop.mapreduce.MiniHadoopClusterManager; import org.apache.hadoop.mapreduce.SleepJob; @@ -90,6 +91,8 @@ public class MapredTestDriver { pgd.addClass("fail", FailJob.class, "a job that always fails"); pgd.addClass("sleep", SleepJob.class, "A job that sleeps at each map and reduce task."); + pgd.addClass("gsleep", GrowingSleepJob.class, + "A sleep job whose mappers create 1MB buffer for every record."); pgd.addClass("timelineperformance", TimelineServicePerformance.class, "A job that launches mappers to test timline service " + "performance."); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org