hadoop-mapreduce-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Ted Yu <yuzhih...@gmail.com>
Subject Re: the options that used to tuning mapreduceV1 is still useful for YARN?
Date Mon, 12 Aug 2013 05:04:36 GMT
In Configuration class, you should be able to find addDeprecation() methods.

Below is the result of quick search where addDeprecation() is called.

    Configuration.addDeprecation("topology.script.file.name",
    Configuration.addDeprecation("topology.script.number.args",
    Configuration.addDeprecation("hadoop.configured.node.mapping",
    Configuration.addDeprecation("topology.node.switch.mapping.impl",
    Configuration.addDeprecation("dfs.df.interval",
    Configuration.addDeprecation("hadoop.native.lib",
    Configuration.addDeprecation("fs.default.name",
    Configuration.addDeprecation("dfs.umaskmode",
./hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
    Configuration.addDeprecation(oldKey, newKey);
./hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
    Configuration.addDeprecation("mapred.temp.dir",
    Configuration.addDeprecation("mapred.local.dir",
    Configuration.addDeprecation("mapred.cluster.map.memory.mb",
    Configuration.addDeprecation("mapred.cluster.reduce.memory.mb",
    Configuration.addDeprecation("mapred.acls.enabled",
    Configuration.addDeprecation("mapred.cluster.max.map.memory.mb",
    Configuration.addDeprecation("mapred.cluster.max.reduce.memory.mb",

Configuration.addDeprecation("mapred.cluster.average.blacklist.threshold",
    Configuration.addDeprecation("hadoop.job.history.location",
    Configuration.addDeprecation(
    Configuration.addDeprecation("mapred.jobtracker.job.history.block.size",

Configuration.addDeprecation("mapred.job.tracker.jobhistory.lru.cache.size",
    Configuration.addDeprecation("mapred.hosts",
    Configuration.addDeprecation("mapred.hosts.exclude",
    Configuration.addDeprecation("mapred.system.dir",
    Configuration.addDeprecation("mapred.max.tracker.blacklists",
    Configuration.addDeprecation("mapred.job.tracker",
    Configuration.addDeprecation("mapred.job.tracker.http.address",
    Configuration.addDeprecation("mapred.job.tracker.handler.count",
    Configuration.addDeprecation("mapred.jobtracker.restart.recover",
    Configuration.addDeprecation("mapred.jobtracker.taskScheduler",
    Configuration.addDeprecation(
    Configuration.addDeprecation("mapred.jobtracker.instrumentation",
    Configuration.addDeprecation("mapred.jobtracker.maxtasks.per.job",
    Configuration.addDeprecation("mapred.heartbeats.in.second",

Configuration.addDeprecation("mapred.job.tracker.persist.jobstatus.active",

Configuration.addDeprecation("mapred.job.tracker.persist.jobstatus.hours",
    Configuration.addDeprecation("mapred.job.tracker.persist.jobstatus.dir",
    Configuration.addDeprecation("mapred.permissions.supergroup",

Configuration.addDeprecation("mapreduce.jobtracker.permissions.supergroup",
    Configuration.addDeprecation("mapred.task.cache.levels",
    Configuration.addDeprecation("mapred.jobtracker.taskalloc.capacitypad",
    Configuration.addDeprecation("mapred.jobinit.threads",
    Configuration.addDeprecation("mapred.tasktracker.expiry.interval",

Configuration.addDeprecation("mapred.job.tracker.retiredjobs.cache.size",
    Configuration.addDeprecation("mapred.job.tracker.retire.jobs",
    Configuration.addDeprecation("mapred.healthChecker.interval",
    Configuration.addDeprecation("mapred.healthChecker.script.args",
    Configuration.addDeprecation("mapred.healthChecker.script.path",
    Configuration.addDeprecation("mapred.healthChecker.script.timeout",
    Configuration.addDeprecation("mapred.local.dir.minspacekill",
    Configuration.addDeprecation("mapred.local.dir.minspacestart",
    Configuration.addDeprecation("mapred.task.tracker.http.address",
    Configuration.addDeprecation("mapred.task.tracker.report.address",
    Configuration.addDeprecation("mapred.task.tracker.task-controller",
    Configuration.addDeprecation("mapred.tasktracker.dns.interface",
    Configuration.addDeprecation("mapred.tasktracker.dns.nameserver",
    Configuration.addDeprecation("mapred.tasktracker.events.batchsize",
    Configuration.addDeprecation("mapred.tasktracker.indexcache.mb",
    Configuration.addDeprecation("mapred.tasktracker.instrumentation",
    Configuration.addDeprecation("mapred.tasktracker.map.tasks.maximum",

Configuration.addDeprecation("mapred.tasktracker.memory_calculator_plugin",

Configuration.addDeprecation("mapred.tasktracker.memorycalculatorplugin",
    Configuration.addDeprecation("mapred.tasktracker.reduce.tasks.maximum",
    Configuration.addDeprecation(
    Configuration.addDeprecation(
    Configuration.addDeprecation("slave.host.name",
    Configuration.addDeprecation("tasktracker.http.threads",
    Configuration.addDeprecation("hadoop.net.static.resolutions",
    Configuration.addDeprecation("local.cache.size",
    Configuration.addDeprecation("tasktracker.contention.tracking",
    Configuration.addDeprecation("job.end.notification.url",
    Configuration.addDeprecation("job.end.retry.attempts",
    Configuration.addDeprecation("job.end.retry.interval",

Configuration.addDeprecation("mapred.committer.job.setup.cleanup.needed",
    Configuration.addDeprecation("mapred.jar",
    Configuration.addDeprecation("mapred.job.id",
    Configuration.addDeprecation("mapred.job.name",
    Configuration.addDeprecation("mapred.job.priority",
    Configuration.addDeprecation("mapred.job.queue.name",
    Configuration.addDeprecation("mapred.job.reuse.jvm.num.tasks",
    Configuration.addDeprecation("mapred.map.tasks",
    Configuration.addDeprecation("mapred.max.tracker.failures",
    Configuration.addDeprecation("mapred.reduce.slowstart.completed.maps",
    Configuration.addDeprecation("mapred.reduce.tasks",
    Configuration.addDeprecation("mapred.skip.on",
    Configuration.addDeprecation("mapred.skip.out.dir",
    Configuration.addDeprecation(
    Configuration.addDeprecation(

Configuration.addDeprecation("mapred.speculative.execution.speculativeCap",
    Configuration.addDeprecation("job.local.dir",
    Configuration.addDeprecation("mapreduce.inputformat.class",
    Configuration.addDeprecation("mapreduce.map.class",
    Configuration.addDeprecation("mapreduce.combine.class",
    Configuration.addDeprecation("mapreduce.reduce.class",
    Configuration.addDeprecation("mapreduce.outputformat.class",
    Configuration.addDeprecation("mapreduce.partitioner.class",
    Configuration.addDeprecation("mapred.job.classpath.archives",
    Configuration.addDeprecation("mapred.job.classpath.files",
    Configuration.addDeprecation("mapred.cache.files",
    Configuration.addDeprecation("mapred.cache.archives",
    Configuration.addDeprecation("mapred.cache.localFiles",
    Configuration.addDeprecation("mapred.cache.localArchives",
    Configuration.addDeprecation("mapred.cache.files.filesizes",
    Configuration.addDeprecation("mapred.cache.archives.filesizes",
    Configuration.addDeprecation("mapred.cache.files.timestamps",
    Configuration.addDeprecation("mapred.cache.archives.timestamps",
    Configuration.addDeprecation("mapred.working.dir",
    Configuration.addDeprecation("user.name",
    Configuration.addDeprecation("mapred.output.key.class",
    Configuration.addDeprecation("mapred.output.value.class",
    Configuration.addDeprecation("mapred.output.value.groupfn.class",
    Configuration.addDeprecation("mapred.output.key.comparator.class",
    Configuration.addDeprecation("io.sort.factor",
    Configuration.addDeprecation("io.sort.mb",
    Configuration.addDeprecation("keep.failed.task.files",
    Configuration.addDeprecation("keep.task.files.pattern",
    Configuration.addDeprecation("mapred.child.tmp",
    Configuration.addDeprecation("mapred.debug.out.lines",
    Configuration.addDeprecation("mapred.merge.recordsBeforeProgress",
    Configuration.addDeprecation("mapred.skip.attempts.to.start.skipping",
    Configuration.addDeprecation("mapred.task.id",
    Configuration.addDeprecation("mapred.task.is.map",
    Configuration.addDeprecation("mapred.task.partition",
    Configuration.addDeprecation("mapred.task.profile",
    Configuration.addDeprecation("mapred.task.profile.maps",
    Configuration.addDeprecation("mapred.task.profile.reduces",
    Configuration.addDeprecation("mapred.task.timeout",
    Configuration.addDeprecation("mapred.tip.id",
    Configuration.addDeprecation("mapred.work.output.dir",
    Configuration.addDeprecation("mapred.userlog.limit.kb",
    Configuration.addDeprecation("mapred.userlog.retain.hours",
    Configuration.addDeprecation("mapred.task.profile.params",
    Configuration.addDeprecation("io.sort.spill.percent",
    Configuration.addDeprecation("map.input.file",
    Configuration.addDeprecation("map.input.length",
    Configuration.addDeprecation("map.input.start",
    Configuration.addDeprecation("mapred.job.map.memory.mb",
    Configuration.addDeprecation("mapred.map.child.env",
    Configuration.addDeprecation("mapred.map.child.java.opts",
    Configuration.addDeprecation("mapred.map.max.attempts",
    Configuration.addDeprecation("mapred.map.task.debug.script",
    Configuration.addDeprecation("mapred.map.tasks.speculative.execution",
    Configuration.addDeprecation("mapred.max.map.failures.percent",
    Configuration.addDeprecation("mapred.skip.map.auto.incr.proc.count",
    Configuration.addDeprecation("mapred.skip.map.max.skip.records",
    Configuration.addDeprecation("min.num.spills.for.combine",
    Configuration.addDeprecation("mapred.compress.map.output",
    Configuration.addDeprecation("mapred.map.output.compression.codec",
    Configuration.addDeprecation("mapred.mapoutput.key.class",
    Configuration.addDeprecation("mapred.mapoutput.value.class",
    Configuration.addDeprecation("map.output.key.field.separator",
    Configuration.addDeprecation("mapred.map.child.log.level",
    Configuration.addDeprecation("mapred.inmem.merge.threshold",
    Configuration.addDeprecation("mapred.job.reduce.input.buffer.percent",

Configuration.addDeprecation("mapred.job.reduce.markreset.buffer.percent",
    Configuration.addDeprecation("mapred.job.reduce.memory.mb",
    Configuration.addDeprecation("mapred.job.reduce.total.mem.bytes",
    Configuration.addDeprecation("mapred.job.shuffle.input.buffer.percent",
    Configuration.addDeprecation("mapred.job.shuffle.merge.percent",
    Configuration.addDeprecation("mapred.max.reduce.failures.percent",
    Configuration.addDeprecation("mapred.reduce.child.env",
    Configuration.addDeprecation("mapred.reduce.child.java.opts",
    Configuration.addDeprecation("mapred.reduce.max.attempts",
    Configuration.addDeprecation("mapred.reduce.parallel.copies",
    Configuration.addDeprecation("mapred.reduce.task.debug.script",

Configuration.addDeprecation("mapred.reduce.tasks.speculative.execution",
    Configuration.addDeprecation("mapred.shuffle.connect.timeout",
    Configuration.addDeprecation("mapred.shuffle.read.timeout",
    Configuration.addDeprecation("mapred.skip.reduce.auto.incr.proc.count",
    Configuration.addDeprecation("mapred.skip.reduce.max.skip.groups",
    Configuration.addDeprecation("mapred.reduce.child.log.level",
    Configuration.addDeprecation("mapreduce.job.counters.limit",
    Configuration.addDeprecation("jobclient.completion.poll.interval",
    Configuration.addDeprecation("jobclient.progress.monitor.poll.interval",
    Configuration.addDeprecation("jobclient.output.filter",
    Configuration.addDeprecation("mapred.submit.replication",
    Configuration.addDeprecation("mapred.used.genericoptionsparser",
    Configuration.addDeprecation("mapred.input.dir",
    Configuration.addDeprecation("mapred.input.pathFilter.class",
    Configuration.addDeprecation("mapred.max.split.size",
    Configuration.addDeprecation("mapred.min.split.size",
    Configuration.addDeprecation("mapred.output.compress",
    Configuration.addDeprecation("mapred.output.compression.codec",
    Configuration.addDeprecation("mapred.output.compression.type",
    Configuration.addDeprecation("mapred.output.dir",
    Configuration.addDeprecation("mapred.seqbinary.output.key.class",
    Configuration.addDeprecation("mapred.seqbinary.output.value.class",
    Configuration.addDeprecation("sequencefile.filter.class",
    Configuration.addDeprecation("sequencefile.filter.regex",
    Configuration.addDeprecation("sequencefile.filter.frequency",
    Configuration.addDeprecation("mapred.input.dir.mappers",
    Configuration.addDeprecation("mapred.input.dir.formats",
    Configuration.addDeprecation("mapred.line.input.format.linespermap",
    Configuration.addDeprecation("mapred.binary.partitioner.left.offset",
    Configuration.addDeprecation("mapred.binary.partitioner.right.offset",
    Configuration.addDeprecation("mapred.text.key.comparator.options",
    Configuration.addDeprecation("mapred.text.key.partitioner.options",
    Configuration.addDeprecation("mapred.mapper.regex.group",
    Configuration.addDeprecation("mapred.mapper.regex",
    Configuration.addDeprecation("create.empty.dir.if.nonexist",
    Configuration.addDeprecation("mapred.data.field.separator",
    Configuration.addDeprecation("map.output.key.value.fields.spec",
    Configuration.addDeprecation("reduce.output.key.value.fields.spec",
    Configuration.addDeprecation("mapred.min.split.size.per.node",
    Configuration.addDeprecation("mapred.min.split.size.per.rack",
    Configuration.addDeprecation("key.value.separator.in.input.line",
    Configuration.addDeprecation("mapred.linerecordreader.maxlength",
    Configuration.addDeprecation("mapred.lazy.output.format",
    Configuration.addDeprecation("mapred.textoutputformat.separator",
    Configuration.addDeprecation("mapred.join.expr",
    Configuration.addDeprecation("mapred.join.keycomparator",
    Configuration.addDeprecation("hadoop.pipes.command-file.keep",
    Configuration.addDeprecation("hadoop.pipes.executable",
    Configuration.addDeprecation("hadoop.pipes.executable.interpretor",
    Configuration.addDeprecation("hadoop.pipes.java.mapper",
    Configuration.addDeprecation("hadoop.pipes.java.recordreader",
    Configuration.addDeprecation("hadoop.pipes.java.recordwriter",
    Configuration.addDeprecation("hadoop.pipes.java.reducer",
    Configuration.addDeprecation("hadoop.pipes.partitioner",
    Configuration.addDeprecation("mapred.pipes.user.inputformat",
    Configuration.addDeprecation("webinterface.private.actions",
    Configuration.addDeprecation("security.task.umbilical.protocol.acl",
    Configuration.addDeprecation("security.job.submission.protocol.acl",
    Configuration.addDeprecation("mapreduce.user.classpath.first",
    Configuration.addDeprecation(JTConfig.JT_MAX_JOB_SPLIT_METAINFO_SIZE,
    Configuration.addDeprecation("mapred.input.dir.recursive",
./hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
    Configuration.addDeprecation("mapred.reducer.sort",
    Configuration.addDeprecation("mapred.reducer.separator",
./hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
    Configuration.addDeprecation("mapred.cache.files.filesizes",
    Configuration.addDeprecation("mapred.cache.files.visibilities",
./hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java

Cheers

On Sun, Aug 11, 2013 at 7:58 PM, ch huang <justlooks@gmail.com> wrote:

> and where i can find the list of deprecation parameter and new parameter ?
>
>
> On Mon, Aug 12, 2013 at 10:43 AM, Harsh J <harsh@cloudera.com> wrote:
>
>> Yes, the deprecation is graceful and older MR1 properties would still
>> work in 2.x at least, although you're recommended to switch over to
>> the new parameters where warned by the Configuration class.
>>
>> On Mon, Aug 12, 2013 at 8:11 AM, ch huang <justlooks@gmail.com> wrote:
>> > or ,YARN framework has it's own tuning & optimization options?
>>
>>
>>
>> --
>> Harsh J
>>
>
>

Mime
View raw message