Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7586017E8A for ; Mon, 27 Apr 2015 21:26:28 +0000 (UTC) Received: (qmail 84616 invoked by uid 500); 27 Apr 2015 21:26:06 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 84422 invoked by uid 500); 27 Apr 2015 21:26:06 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 81848 invoked by uid 99); 27 Apr 2015 21:26:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Apr 2015 21:26:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3557EE30F2; Mon, 27 Apr 2015 21:26:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zjshen@apache.org To: common-commits@hadoop.apache.org Date: Mon, 27 Apr 2015 21:26:48 -0000 Message-Id: <8a2be9daee1e4f509a87ea9e4da7c9f4@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [45/50] [abbrv] hadoop git commit: MAPREDUCE-6057. Remove obsolete entries from mapred-default.xml (Ray Chiang via aw) MAPREDUCE-6057. Remove obsolete entries from mapred-default.xml (Ray Chiang via aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3044c8da Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3044c8da Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3044c8da Branch: refs/heads/YARN-2928 Commit: 3044c8da56aed66ae0eb48988def9e6693be60c2 Parents: bd8af28 Author: Allen Wittenauer Authored: Sun Apr 26 20:31:40 2015 -0700 Committer: Zhijie Shen Committed: Mon Apr 27 14:18:54 2015 -0700 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/mapred/MapTask.java | 5 ++-- .../apache/hadoop/mapreduce/MRJobConfig.java | 3 ++ .../src/main/resources/mapred-default.xml | 31 -------------------- .../resources/job_1329348432655_0001_conf.xml | 4 --- .../src/main/data/2jobs2min-rumen-jh.json | 6 ---- 6 files changed, 9 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044c8da/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index f895034..dca42c4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -120,6 +120,9 @@ Trunk (Unreleased) MAPREDUCE-6260. Convert site documentation to markdown (Masatake Iwasaki via aw) + MAPREDUCE-6057. Remove obsolete entries from mapred-default.xml + (Ray Chiang via aw) + BUG FIXES MAPREDUCE-6191. Improve clearing stale state of Java serialization http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044c8da/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index c4957b7..a523291 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -978,8 +978,9 @@ public class MapTask extends Task { throw new IOException( "Invalid \"" + JobContext.IO_SORT_MB + "\": " + sortmb); } - sorter = ReflectionUtils.newInstance(job.getClass("map.sort.class", - QuickSort.class, IndexedSorter.class), job); + sorter = ReflectionUtils.newInstance(job.getClass( + MRJobConfig.MAP_SORT_CLASS, QuickSort.class, + IndexedSorter.class), job); // buffers and accounting int maxMemUsage = sortmb << 20; maxMemUsage -= maxMemUsage % METASIZE; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044c8da/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 8176efd..bc31bb5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -28,6 +28,9 @@ import org.apache.hadoop.yarn.util.Apps; @InterfaceStability.Evolving public interface MRJobConfig { + // Used by MapTask + public static final String MAP_SORT_CLASS = "map.sort.class"; + // Put all of the attribute names in here so that Job and JobContext are // consistent. public static final String INPUT_FORMAT_CLASS_ATTR = "mapreduce.job.inputformat.class"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044c8da/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index b5031f7..2b25b59 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -58,13 +58,6 @@ - mapreduce.local.clientfactory.class.name - org.apache.hadoop.mapred.LocalClientFactory - This the client factory that is responsible for - creating local job runner client - - - mapreduce.job.maps 2 The default number of map tasks per job. @@ -891,30 +884,6 @@ - mapreduce.map.skip.proc.count.autoincr - true - The flag which if set to true, - SkipBadRecords.COUNTER_MAP_PROCESSED_RECORDS is incremented - by MapRunner after invoking the map function. This value must be set to - false for applications which process the records asynchronously - or buffer the input records. For example streaming. - In such cases applications should increment this counter on their own. - - - - - mapreduce.reduce.skip.proc.count.autoincr - true - The flag which if set to true, - SkipBadRecords.COUNTER_REDUCE_PROCESSED_GROUPS is incremented - by framework after invoking the reduce function. This value must be set to - false for applications which process the records asynchronously - or buffer the input records. For example streaming. - In such cases applications should increment this counter on their own. - - - - mapreduce.job.skip.outdir If no value is specified here, the skipped records are http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044c8da/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml index 6d1c175..c469f78 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml @@ -93,7 +93,6 @@ mapreduce.reduce.input.buffer.percent0.0 mapreduce.map.output.compress.codecorg.apache.hadoop.io.compress.DefaultCodec yarn.resourcemanager.delegation-token.keepalive-time-ms300000 -mapreduce.map.skip.proc.count.autoincrtrue dfs.datanode.directoryscan.threads1 mapreduce.jobtracker.addresslocal mapreduce.cluster.local.dir${hadoop.tmp.dir}/mapred/local @@ -197,7 +196,6 @@ dfs.block.access.key.update.interval600 mapreduce.jobhistory.move.interval-ms30000 dfs.datanode.dns.interfacedefault -mapreduce.reduce.skip.proc.count.autoincrtrue dfs.namenode.backup.http-address0.0.0.0:50105 yarn.nodemanager.container-monitor.interval-ms3000 mapred.reducer.new-apitrue @@ -290,7 +288,6 @@ yarn.app.mapreduce.am.command-opts-Xmx500m mapreduce.admin.user.envLD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native dfs.namenode.checkpoint.edits.dir${dfs.namenode.checkpoint.dir} -mapreduce.local.clientfactory.class.nameorg.apache.hadoop.mapred.LocalClientFactory hadoop.common.configuration.version0.23.0 mapreduce.tasktracker.dns.interfacedefault io.serializationsorg.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization @@ -380,7 +377,6 @@ fs.AbstractFileSystem.viewfs.implorg.apache.hadoop.fs.viewfs.ViewFs yarn.resourcemanager.resource-tracker.client.thread-count50 mapreduce.tasktracker.dns.nameserverdefault -mapreduce.clientfactory.class.nameorg.apache.hadoop.mapred.YarnClientFactory mapreduce.map.output.compressfalse mapreduce.job.counters.limit120 dfs.datanode.ipc.address0.0.0.0:50020 http://git-wip-us.apache.org/repos/asf/hadoop/blob/3044c8da/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json index f54c26f..4be3e08 100644 --- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json +++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json @@ -4645,7 +4645,6 @@ "dfs.ha.log-roll.period" : "120", "mapreduce.reduce.input.buffer.percent" : "0.0", "mapreduce.map.output.compress.codec" : "org.apache.hadoop.io.compress.SnappyCodec", - "mapreduce.map.skip.proc.count.autoincr" : "true", "dfs.client.failover.sleep.base.millis" : "500", "dfs.datanode.directoryscan.threads" : "1", "mapreduce.jobtracker.address" : "neededForHive:999999", @@ -4765,7 +4764,6 @@ "dfs.namenode.backup.address" : "0.0.0.0:50100", "hadoop.util.hash.type" : "murmur", "dfs.block.access.key.update.interval" : "600", - "mapreduce.reduce.skip.proc.count.autoincr" : "true", "dfs.datanode.dns.interface" : "default", "dfs.datanode.use.datanode.hostname" : "false", "mapreduce.job.output.key.class" : "org.apache.hadoop.io.Text", @@ -4875,7 +4873,6 @@ "mapreduce.reduce.shuffle.read.timeout" : "180000", "mapreduce.admin.user.env" : "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native", "yarn.app.mapreduce.am.command-opts" : " -Xmx1238932873", - "mapreduce.local.clientfactory.class.name" : "org.apache.hadoop.mapred.LocalClientFactory", "dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}", "fs.permissions.umask-mode" : "022", "dfs.client.domain.socket.data.traffic" : "false", @@ -9754,7 +9751,6 @@ "dfs.ha.log-roll.period" : "120", "mapreduce.reduce.input.buffer.percent" : "0.0", "mapreduce.map.output.compress.codec" : "org.apache.hadoop.io.compress.SnappyCodec", - "mapreduce.map.skip.proc.count.autoincr" : "true", "dfs.client.failover.sleep.base.millis" : "500", "dfs.datanode.directoryscan.threads" : "1", "mapreduce.jobtracker.address" : "neededForHive:999999", @@ -9874,7 +9870,6 @@ "dfs.namenode.backup.address" : "0.0.0.0:50100", "hadoop.util.hash.type" : "murmur", "dfs.block.access.key.update.interval" : "600", - "mapreduce.reduce.skip.proc.count.autoincr" : "true", "dfs.datanode.dns.interface" : "default", "dfs.datanode.use.datanode.hostname" : "false", "mapreduce.job.output.key.class" : "org.apache.hadoop.io.Text", @@ -9984,7 +9979,6 @@ "mapreduce.reduce.shuffle.read.timeout" : "180000", "mapreduce.admin.user.env" : "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native", "yarn.app.mapreduce.am.command-opts" : " -Xmx1238932873", - "mapreduce.local.clientfactory.class.name" : "org.apache.hadoop.mapred.LocalClientFactory", "dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}", "fs.permissions.umask-mode" : "022", "dfs.client.domain.socket.data.traffic" : "false",