hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [6/6] hadoop git commit: YARN-6608. Backport all SLS improvements from trunk to branch-2. (Carlo Curino via wangda)
Date Wed, 18 Oct 2017 02:45:28 GMT
YARN-6608. Backport all SLS improvements from trunk to branch-2. (Carlo Curino via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d894f910
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d894f910
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d894f910

Branch: refs/heads/branch-2
Commit: d894f910d09dd219c540b66b6bd7d15198f4c01b
Parents: 5e7491d
Author: Wangda Tan <wangda@apache.org>
Authored: Tue Oct 17 19:45:09 2017 -0700
Committer: Wangda Tan <wangda@apache.org>
Committed: Tue Oct 17 19:45:09 2017 -0700

----------------------------------------------------------------------
 .../hadoop/metrics2/source/JvmMetrics.java      |  10 +
 .../org/apache/hadoop/tools/rumen/TaskInfo.java |  29 +-
 hadoop-tools/hadoop-sls/pom.xml                 |  48 +-
 .../hadoop-sls/src/main/assemblies/sls.xml      |   4 +-
 hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh  |  31 +-
 .../src/main/data/2jobs2min-rumen-jh.json       | 705 +-------------
 .../src/main/html/js/thirdparty/jquery.js       |   2 +-
 .../hadoop/yarn/sls/ReservationClientUtil.java  |  78 ++
 .../hadoop/yarn/sls/RumenToSLSConverter.java    |   8 +-
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   | 932 ++++++++++++------
 .../hadoop/yarn/sls/appmaster/AMSimulator.java  | 259 +++--
 .../yarn/sls/appmaster/MRAMSimulator.java       | 230 ++---
 .../hadoop/yarn/sls/conf/SLSConfiguration.java  |  15 +
 .../yarn/sls/nodemanager/NMSimulator.java       |   1 -
 .../sls/resourcemanager/MockAMLauncher.java     | 120 +++
 .../sls/scheduler/FairSchedulerMetrics.java     | 328 +++----
 .../sls/scheduler/ResourceSchedulerWrapper.java | 973 -------------------
 .../sls/scheduler/SLSCapacityScheduler.java     | 680 +++----------
 .../yarn/sls/scheduler/SLSFairScheduler.java    | 346 +++++++
 .../yarn/sls/scheduler/SchedulerMetrics.java    | 605 +++++++++++-
 .../yarn/sls/scheduler/SchedulerWrapper.java    |  27 +-
 .../hadoop/yarn/sls/scheduler/TaskRunner.java   |   9 +-
 .../hadoop/yarn/sls/scheduler/Tracker.java      |  46 +
 .../hadoop/yarn/sls/synthetic/SynthJob.java     | 306 ++++++
 .../yarn/sls/synthetic/SynthJobClass.java       | 180 ++++
 .../sls/synthetic/SynthTraceJobProducer.java    | 319 ++++++
 .../hadoop/yarn/sls/synthetic/SynthUtils.java   | 101 ++
 .../yarn/sls/synthetic/SynthWorkload.java       | 121 +++
 .../hadoop/yarn/sls/synthetic/package-info.java |  22 +
 .../apache/hadoop/yarn/sls/utils/SLSUtils.java  |  61 +-
 .../apache/hadoop/yarn/sls/web/SLSWebApp.java   |  45 +-
 .../src/site/markdown/SchedulerLoadSimulator.md | 151 ++-
 .../hadoop/yarn/sls/BaseSLSRunnerTest.java      | 151 +++
 .../sls/TestReservationSystemInvariants.java    |  77 ++
 .../apache/hadoop/yarn/sls/TestSLSRunner.java   |  95 +-
 .../hadoop/yarn/sls/TestSynthJobGeneration.java |  96 ++
 .../yarn/sls/appmaster/TestAMSimulator.java     |  89 +-
 .../yarn/sls/nodemanager/TestNMSimulator.java   |  32 +-
 .../yarn/sls/scheduler/TestTaskRunner.java      |   2 +-
 .../hadoop/yarn/sls/utils/TestSLSUtils.java     |  30 +
 .../hadoop/yarn/sls/web/TestSLSWebApp.java      |  28 +-
 .../src/test/resources/capacity-scheduler.xml   |  10 +
 .../src/test/resources/exit-invariants.txt      |   8 +
 .../src/test/resources/fair-scheduler.xml       |   8 +-
 .../hadoop-sls/src/test/resources/inputsls.json |  55 ++
 .../src/test/resources/log4j.properties         |  19 +
 .../hadoop-sls/src/test/resources/nodes.json    |  84 ++
 .../src/test/resources/ongoing-invariants.txt   |  54 +
 .../src/test/resources/sls-runner.xml           |   6 +-
 .../hadoop-sls/src/test/resources/syn.json      |  53 +
 .../hadoop-sls/src/test/resources/yarn-site.xml |  19 +-
 .../hadoop-yarn-server-resourcemanager/pom.xml  |   1 +
 .../invariants/InvariantViolationException.java |  35 +
 .../monitor/invariants/InvariantsChecker.java   |  96 ++
 .../invariants/MetricsInvariantChecker.java     | 195 ++++
 .../ReservationInvariantsChecker.java           |  63 ++
 .../monitor/invariants/package-info.java        |  22 +
 .../invariants/TestMetricsInvariantChecker.java |  99 ++
 .../src/test/resources/invariants.txt           |  54 +
 59 files changed, 5172 insertions(+), 3101 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
index caba170..c6369cd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
@@ -27,6 +27,7 @@ import java.lang.management.GarbageCollectorMXBean;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.log.metrics.EventCounter;
 import org.apache.hadoop.metrics2.MetricsCollector;
@@ -59,6 +60,15 @@ public class JvmMetrics implements MetricsSource {
     }
   }
 
+  @VisibleForTesting
+  public synchronized void registerIfNeeded(){
+    // during tests impl might exist, but is not registered
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    if (ms.getSource("JvmMetrics") == null) {
+      ms.register(JvmMetrics.name(), JvmMetrics.description(), this);
+    }
+  }
+
   static final float M = 1024*1024;
   static public final float MEMORY_MAX_UNLIMITED_MB = -1;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
index 9aa6373..6159f85 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
@@ -23,21 +23,37 @@ public class TaskInfo {
   private final long bytesOut;
   private final int recsOut;
   private final long maxMemory;
+  private final long maxVcores;
   private final ResourceUsageMetrics metrics;
 
+
+  public TaskInfo(long bytesIn, int recsIn, long bytesOut, int recsOut,
+                  long maxMemory) {
+    this(bytesIn, recsIn, bytesOut, recsOut, maxMemory, 1,
+        new ResourceUsageMetrics());
+  }
+
   public TaskInfo(long bytesIn, int recsIn, long bytesOut, int recsOut,
-      long maxMemory) {
-    this(bytesIn, recsIn, bytesOut, recsOut, maxMemory, 
+                  long maxMemory, ResourceUsageMetrics
+                      metrics) {
+    this(bytesIn, recsIn, bytesOut, recsOut, maxMemory, 1, metrics);
+  }
+
+  public TaskInfo(long bytesIn, int recsIn, long bytesOut, int recsOut,
+      long maxMemory, long maxVcores) {
+    this(bytesIn, recsIn, bytesOut, recsOut, maxMemory, maxVcores,
          new ResourceUsageMetrics());
   }
   
   public TaskInfo(long bytesIn, int recsIn, long bytesOut, int recsOut,
-                  long maxMemory, ResourceUsageMetrics metrics) {
+                  long maxMemory, long maxVcores, ResourceUsageMetrics
+                      metrics) {
     this.bytesIn = bytesIn;
     this.recsIn = recsIn;
     this.bytesOut = bytesOut;
     this.recsOut = recsOut;
     this.maxMemory = maxMemory;
+    this.maxVcores = maxVcores;
     this.metrics = metrics;
   }
 
@@ -79,6 +95,13 @@ public class TaskInfo {
   }
 
   /**
+   * @return Vcores used by the task.
+   */
+  public long getTaskVCores() {
+    return maxVcores;
+  }
+
+  /**
    * @return Resource usage metrics
    */
   public ResourceUsageMetrics getResourceUsageMetrics() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-tools/hadoop-sls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index 3accc08..4b7a3e2 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -22,7 +22,6 @@
     <version>2.9.0-SNAPSHOT</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
-  <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-sls</artifactId>
   <version>2.9.0-SNAPSHOT</version>
   <description>Apache Hadoop Scheduler Load Simulator</description>
@@ -70,9 +69,34 @@
       <artifactId>jetty-util</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.4</version>
+    </dependency>
   </dependencies>
 
   <build>
+    <resources>
+      <resource>
+      <directory>src/main/</directory>
+      <includes>
+        <include>html/simulate.html.template</include>
+        <include>html/simulate.info.html.template</include>
+        <include>html/track.html.template</include>
+        <include>html/css/bootstrap-responsive.min.css</include>
+        <include>html/css/bootstrap.min.css</include>
+        <include>html/js/thirdparty/bootstrap.min.js</include>
+        <include>html/js/thirdparty/d3.v3.js</include>
+        <include>html/js/thirdparty/d3-LICENSE</include>
+        <include>html/js/thirdparty/jquery.js</include>
+      </includes>
+      </resource>
+    </resources>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
@@ -113,6 +137,11 @@
             <exclude>src/test/resources/simulate.html.template</exclude>
             <exclude>src/test/resources/simulate.info.html.template</exclude>
             <exclude>src/test/resources/track.html.template</exclude>
+            <exclude>src/test/resources/syn.json</exclude>
+            <exclude>src/test/resources/inputsls.json</exclude>
+            <exclude>src/test/resources/nodes.json</exclude>
+            <exclude>src/test/resources/exit-invariants.txt</exclude>
+            <exclude>src/test/resources/ongoing-invariants.txt</exclude>
           </excludes>
         </configuration>
       </plugin>
@@ -177,6 +206,23 @@
               </execution>
             </executions>
           </plugin>
+       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- referenced by a built-in command -->
+              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
         </plugins>
       </build>
     </profile>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml b/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml
index 5b09c81..085a5d8 100644
--- a/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml
+++ b/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml
@@ -15,7 +15,9 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
-<assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
   <id>sls</id>
   <formats>
     <format>dir</format>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index b580716..4a10938 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -16,7 +16,9 @@
 ###############################################################################
 printUsage() {
   echo "Usage: slsrun.sh <OPTIONS>"
-  echo "                 --input-rumen|--input-sls=<FILE1,FILE2,...>"
+  echo "                 --tracetype=<SYNTH | SLS | RUMEN>"
+  echo "                 --tracelocation=<FILE1,FILE2,...>"
+  echo "                 (deprecated --input-rumen=<FILE1,FILE2,...>  | --input-sls=<FILE1,FILE2,...>)"
   echo "                 --output-dir=<SLS_SIMULATION_OUTPUT_DIRECTORY>"
   echo "                 [--nodes=<SLS_NODES_FILE>]"
   echo "                 [--track-jobs=<JOBID1,JOBID2,...>]"
@@ -28,6 +30,12 @@ parseArgs() {
   for i in $*
   do
     case $i in
+    --tracetype=*)
+      tracetype=${i#*=}
+      ;;
+    --tracelocation=*)
+      tracelocation=${i#*=}
+      ;;
     --input-rumen=*)
       inputrumen=${i#*=}
       ;;
@@ -55,8 +63,8 @@ parseArgs() {
     esac
   done
 
-  if [[ "${inputrumen}" == "" && "${inputsls}" == "" ]] ; then
-    echo "Either --input-rumen or --input-sls must be specified"
+  if [[ "${inputrumen}" == "" && "${inputsls}" == "" && "${tracetype}" == "" ]] ; then
+    echo "Either --input-rumen or --input-sls or --tracetype must be specified"
     echo
     printUsage
     exit 1
@@ -82,12 +90,23 @@ calculateClasspath() {
 }
 ###############################################################################
 runSimulation() {
-  if [[ "${inputsls}" == "" ]] ; then
-    args="-inputrumen ${inputrumen}"
-  else
+  if [[ "${tracetype}" != "" ]] ; then
+    args="${args} -tracetype ${tracetype}"
+    args="${args} -tracelocation ${tracelocation}"
+  fi
+
+  if [[ "${nodes}" != "" ]] ; then
+    args="${args} -nodes ${nodes}"
+  fi
+
+  if [[ "${inputsls}" != "" ]] ; then
     args="-inputsls ${inputsls}"
   fi
 
+  if [[ "${inputrumen}" != "" ]] ; then
+    args="-inputrumen ${inputrumen}"
+  fi
+
   args="${args} -output ${outputdir}"
 
   if [[ "${nodes}" != "" ]] ; then

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
index be868af..8298d60 100644
--- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
+++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
@@ -4545,22 +4545,19 @@
     "hadoop.ssl.keystores.factory.class" : "org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory",
     "hadoop.http.authentication.kerberos.keytab" : "${user.home}/hadoop.keytab",
     "yarn.nodemanager.keytab" : "/etc/krb5.keytab",
-    "s3.blocksize" : "67108864",
     "mapreduce.task.io.sort.factor" : "10",
     "yarn.nodemanager.disk-health-checker.interval-ms" : "120000",
-    "mapreduce.job.working.dir" : "hdfs://a2115.smile.com:8020/user/jenkins",
+    "mapreduce.job.working.dir" : "hdfs://a2115.smile.com:9820/user/jenkins",
     "yarn.admin.acl" : "*",
     "mapreduce.job.speculative.speculativecap" : "0.1",
     "dfs.namenode.num.checkpoints.retained" : "2",
     "dfs.namenode.delegation.token.renew-interval" : "86400000",
     "yarn.nodemanager.resource.memory-mb" : "8192",
     "io.map.index.interval" : "128",
-    "s3.client-write-packet-size" : "65536",
     "mapreduce.task.files.preserve.failedtasks" : "false",
     "dfs.namenode.http-address" : "a2115.smile.com:20101",
     "ha.zookeeper.session-timeout.ms" : "5000",
     "hadoop.hdfs.configuration.version" : "1",
-    "s3.replication" : "3",
     "dfs.datanode.balance.bandwidthPerSec" : "1048576",
     "mapreduce.reduce.shuffle.connect.timeout" : "180000",
     "hadoop.ssl.enabled" : "false",
@@ -4584,10 +4581,9 @@
     "dfs.namenode.edits.dir" : "${dfs.namenode.name.dir}",
     "ha.health-monitor.sleep-after-disconnect.ms" : "1000",
     "dfs.encrypt.data.transfer" : "false",
-    "dfs.datanode.http.address" : "0.0.0.0:50075",
+    "dfs.datanode.http.address" : "0.0.0.0:9864",
     "mapreduce.terasort.num-rows" : "400000000",
     "mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper",
-    "mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12",
     "dfs.namenode.write.stale.datanode.ratio" : "0.5f",
     "dfs.client.use.datanode.hostname" : "false",
     "yarn.acl.enable" : "true",
@@ -4604,7 +4600,6 @@
     "mapreduce.reduce.log.level" : "INFO",
     "yarn.log-aggregation-enable" : "false",
     "dfs.datanode.sync.behind.writes" : "false",
-    "mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst",
     "dfs.https.server.keystore.resource" : "ssl-server.xml",
     "hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn",
     "dfs.namenode.replication.min" : "1",
@@ -4627,7 +4622,6 @@
     "hadoop.ssl.client.conf" : "ssl-client.xml",
     "dfs.namenode.safemode.threshold-pct" : "0.999f",
     "mapreduce.tasktracker.local.dir.minspacekill" : "0",
-    "mapreduce.jobtracker.retiredjobs.cache.size" : "1000",
     "dfs.blocksize" : "134217728",
     "yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler",
     "mapreduce.job.reduce.slowstart.completedmaps" : "0.8",
@@ -4637,14 +4631,12 @@
     "mapreduce.job.user.name" : "jenkins",
     "mapreduce.tasktracker.outofband.heartbeat" : "false",
     "io.native.lib.available" : "true",
-    "mapreduce.jobtracker.persist.jobstatus.hours" : "0",
     "dfs.client-write-packet-size" : "65536",
     "mapreduce.client.progressmonitor.pollinterval" : "1000",
     "dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name",
     "dfs.ha.log-roll.period" : "120",
     "mapreduce.reduce.input.buffer.percent" : "0.0",
     "mapreduce.map.output.compress.codec" : "org.apache.hadoop.io.compress.SnappyCodec",
-    "mapreduce.map.skip.proc-count.auto-incr" : "true",
     "dfs.client.failover.sleep.base.millis" : "500",
     "dfs.datanode.directoryscan.threads" : "1",
     "mapreduce.jobtracker.address" : "neededForHive:999999",
@@ -4656,13 +4648,12 @@
     "yarn.scheduler.fair.preemption" : "true",
     "mapreduce.reduce.shuffle.parallelcopies" : "5",
     "yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
-    "mapreduce.jobtracker.heartbeats.in.second" : "100",
     "mapreduce.job.maxtaskfailures.per.tracker" : "3",
     "ipc.client.connection.maxidletime" : "10000",
     "mapreduce.shuffle.ssl.enabled" : "false",
     "dfs.namenode.invalidate.work.pct.per.iteration" : "0.32f",
     "dfs.blockreport.intervalMsec" : "21600000",
-    "fs.s3.sleepTimeSeconds" : "10",
+    "fs.s3n.sleepTimeSeconds" : "10",
     "dfs.namenode.replication.considerLoad" : "true",
     "dfs.client.block.write.retries" : "3",
     "hadoop.ssl.server.conf" : "ssl-server.xml",
@@ -4675,8 +4666,6 @@
     "dfs.replication" : "3",
     "ipc.client.tcpnodelay" : "false",
     "dfs.namenode.accesstime.precision" : "3600000",
-    "s3.stream-buffer-size" : "4096",
-    "mapreduce.jobtracker.tasktracker.maxblacklists" : "4",
     "dfs.client.read.shortcircuit.skip.checksum" : "false",
     "mapreduce.job.jvm.numtasks" : "1",
     "mapreduce.task.io.sort.mb" : "100",
@@ -4693,14 +4682,12 @@
     "kfs.stream-buffer-size" : "4096",
     "dfs.ha.tail-edits.period" : "60",
     "hadoop.security.authentication" : "simple",
-    "fs.s3.buffer.dir" : "${hadoop.tmp.dir}/s3",
+    "fs.s3n.buffer.dir" : "${hadoop.tmp.dir}/s3n",
     "rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
-    "mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler",
     "yarn.app.mapreduce.am.job.task.listener.thread-count" : "30",
     "dfs.namenode.avoid.read.stale.datanode" : "false",
     "mapreduce.job.reduces" : "0",
     "mapreduce.map.sort.spill.percent" : "0.8",
-    "dfs.client.file-block-storage-locations.timeout" : "60",
     "dfs.datanode.drop.cache.behind.writes" : "false",
     "mapreduce.job.end-notification.retry.interval" : "1",
     "mapreduce.job.maps" : "96",
@@ -4722,7 +4709,6 @@
     "dfs.namenode.replication.interval" : "3",
     "mapreduce.task.skip.start.attempts" : "2",
     "dfs.namenode.https-address" : "a2115.smile.com:20102",
-    "mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo",
     "ipc.client.kill.max" : "10",
     "dfs.ha.automatic-failover.enabled" : "false",
     "mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab",
@@ -4745,11 +4731,9 @@
     "io.mapfile.bloom.size" : "1048576",
     "yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
     "mapreduce.map.maxattempts" : "4",
-    "mapreduce.jobtracker.jobhistory.block.size" : "3145728",
     "yarn.log-aggregation.retain-seconds" : "-1",
     "yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000",
     "ftp.replication" : "3",
-    "mapreduce.jobtracker.http.address" : "0.0.0.0:50030",
     "yarn.nodemanager.health-checker.script.timeout-ms" : "1200000",
     "mapreduce.jobhistory.address" : "a2115.smile.com:10020",
     "mapreduce.jobtracker.taskcache.levels" : "2",
@@ -4763,7 +4747,6 @@
     "dfs.namenode.backup.address" : "0.0.0.0:50100",
     "hadoop.util.hash.type" : "murmur",
     "dfs.block.access.key.update.interval" : "600",
-    "mapreduce.reduce.skip.proc-count.auto-incr" : "true",
     "dfs.datanode.dns.interface" : "default",
     "dfs.datanode.use.datanode.hostname" : "false",
     "mapreduce.job.output.key.class" : "org.apache.hadoop.io.Text",
@@ -4777,7 +4760,7 @@
     "mapreduce.job.dir" : "/user/jenkins/.staging/job_1369942127770_1205",
     "io.map.index.skip" : "0",
     "net.topology.node.switch.mapping.impl" : "org.apache.hadoop.net.ScriptBasedMapping",
-    "fs.s3.maxRetries" : "4",
+    "fs.s3n.maxRetries" : "4",
     "ha.failover-controller.new-active.rpc-timeout.ms" : "60000",
     "s3native.client-write-packet-size" : "65536",
     "yarn.resourcemanager.amliveliness-monitor.interval-ms" : "1000",
@@ -4800,7 +4783,6 @@
     "dfs.datanode.directoryscan.interval" : "21600",
     "yarn.resourcemanager.address" : "a2115.smile.com:8032",
     "yarn.nodemanager.health-checker.interval-ms" : "600000",
-    "dfs.client.file-block-storage-locations.num-threads" : "10",
     "yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs" : "86400",
     "mapreduce.reduce.markreset.buffer.percent" : "0.0",
     "hadoop.security.group.mapping.ldap.directory.search.timeout" : "10000",
@@ -4813,7 +4795,7 @@
     "ftp.stream-buffer-size" : "4096",
     "dfs.namenode.avoid.write.stale.datanode" : "false",
     "hadoop.security.group.mapping.ldap.search.attr.member" : "member",
-    "mapreduce.output.fileoutputformat.outputdir" : "hdfs://a2115.smile.com:8020/user/jenkins/tera-gen-1",
+    "mapreduce.output.fileoutputformat.outputdir" : "hdfs://a2115.smile.com:9820/user/jenkins/tera-gen-1",
     "dfs.blockreport.initialDelay" : "0",
     "yarn.nm.liveness-monitor.expiry-interval-ms" : "600000",
     "hadoop.http.authentication.token.validity" : "36000",
@@ -4839,18 +4821,16 @@
     "yarn.scheduler.maximum-allocation-mb" : "8192",
     "yarn.nodemanager.heartbeat.interval-ms" : "1000",
     "mapreduce.job.userlog.retain.hours" : "24",
-    "dfs.namenode.secondary.http-address" : "0.0.0.0:50090",
+    "dfs.namenode.secondary.http-address" : "0.0.0.0:9868",
     "mapreduce.task.timeout" : "600000",
     "mapreduce.framework.name" : "yarn",
     "ipc.client.idlethreshold" : "4000",
     "ftp.bytes-per-checksum" : "512",
     "ipc.server.tcpnodelay" : "false",
     "dfs.namenode.stale.datanode.interval" : "30000",
-    "s3.bytes-per-checksum" : "512",
     "mapreduce.job.speculative.slowtaskthreshold" : "1.0",
     "yarn.nodemanager.localizer.cache.target-size-mb" : "10240",
     "yarn.nodemanager.remote-app-log-dir" : "/tmp/logs",
-    "fs.s3.block.size" : "67108864",
     "mapreduce.job.queuename" : "sls_queue_1",
     "dfs.client.failover.connection.retries" : "0",
     "hadoop.rpc.protection" : "authentication",
@@ -4859,7 +4839,7 @@
     "hadoop.security.auth_to_local" : "DEFAULT",
     "dfs.secondary.namenode.kerberos.internal.spnego.principal" : "${dfs.web.authentication.kerberos.principal}",
     "ftp.client-write-packet-size" : "65536",
-    "fs.defaultFS" : "hdfs://a2115.smile.com:8020",
+    "fs.defaultFS" : "hdfs://a2115.smile.com:9820",
     "yarn.nodemanager.address" : "0.0.0.0:0",
     "yarn.scheduler.fair.assignmultiple" : "true",
     "yarn.resourcemanager.scheduler.client.thread-count" : "50",
@@ -4873,7 +4853,6 @@
     "mapreduce.reduce.shuffle.read.timeout" : "180000",
     "mapreduce.admin.user.env" : "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native",
     "yarn.app.mapreduce.am.command-opts" : " -Xmx1238932873",
-    "mapreduce.local.clientfactory.class.name" : "org.apache.hadoop.mapred.LocalClientFactory",
     "dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}",
     "fs.permissions.umask-mode" : "022",
     "dfs.client.domain.socket.data.traffic" : "false",
@@ -4904,15 +4883,14 @@
     "fs.s3n.block.size" : "67108864",
     "fs.ftp.host" : "0.0.0.0",
     "hadoop.security.group.mapping" : "org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback",
-    "dfs.datanode.address" : "0.0.0.0:50010",
+    "dfs.datanode.address" : "0.0.0.0:9866",
     "mapreduce.map.skip.maxrecords" : "0",
-    "dfs.datanode.https.address" : "0.0.0.0:50475",
+    "dfs.datanode.https.address" : "0.0.0.0:9865",
     "file.replication" : "1",
     "yarn.resourcemanager.resource-tracker.address" : "a2115.smile.com:8031",
     "dfs.datanode.drop.cache.behind.reads" : "false",
     "hadoop.fuse.connection.timeout" : "300",
     "hadoop.work.around.non.threadsafe.getpwuid" : "false",
-    "mapreduce.jobtracker.restart.recover" : "false",
     "hadoop.tmp.dir" : "/tmp/hadoop-${user.name}",
     "mapreduce.output.fileoutputformat.compress" : "false",
     "mapreduce.tasktracker.indexcache.mb" : "10",
@@ -4922,10 +4900,8 @@
     "hadoop.kerberos.kinit.command" : "kinit",
     "dfs.datanode.du.reserved" : "0",
     "dfs.namenode.fs-limits.max-blocks-per-file" : "1048576",
-    "dfs.webhdfs.enabled" : "false",
     "file.bytes-per-checksum" : "512",
     "mapreduce.task.profile.reduces" : "0-2",
-    "mapreduce.jobtracker.handler.count" : "10",
     "dfs.client.block.write.replace-datanode-on-failure.enable" : "true",
     "mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text",
     "yarn.dispatcher.exit-on-error" : "true",
@@ -4943,7 +4919,6 @@
     "mapreduce.shuffle.ssl.file.buffer.size" : "65536",
     "dfs.permissions.superusergroup" : "supergroup",
     "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240",
-    "mapreduce.jobtracker.expire.trackers.interval" : "600000",
     "mapreduce.cluster.acls.enabled" : "false",
     "yarn.nodemanager.remote-app-log-dir-suffix" : "logs",
     "ha.failover-controller.graceful-fence.connection.retries" : "1",
@@ -4953,8 +4928,6 @@
     "io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local",
     "dfs.namenode.checkpoint.check.period" : "60",
     "yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000",
-    "mapreduce.jobtracker.maxtasks.perjob" : "-1",
-    "mapreduce.jobtracker.jobhistory.lru.cache.size" : "5",
     "file.blocksize" : "67108864",
     "tfile.io.chunk.size" : "1048576",
     "mapreduce.job.acl-modify-job" : " ",
@@ -4974,7 +4947,7 @@
     "yarn.resourcemanager.resource-tracker.client.thread-count" : "50",
     "mapreduce.tasktracker.dns.nameserver" : "default",
     "mapreduce.map.output.compress" : "true",
-    "dfs.datanode.ipc.address" : "0.0.0.0:50020",
+    "dfs.datanode.ipc.address" : "0.0.0.0:9867",
     "hadoop.ssl.require.client.cert" : "false",
     "yarn.nodemanager.delete.debug-delay-sec" : "0",
     "dfs.datanode.max.transfer.threads" : "4096"
@@ -5105,7 +5078,8 @@
   "clusterReduceMB" : -1,
   "jobMapMB" : 200,
   "jobReduceMB" : 200
-} {
+}
+{
   "priority" : "NORMAL",
   "jobID" : "job_1369942127770_1206",
   "user" : "jenkins",
@@ -9652,22 +9626,19 @@
     "hadoop.ssl.keystores.factory.class" : "org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory",
     "hadoop.http.authentication.kerberos.keytab" : "${user.home}/hadoop.keytab",
     "yarn.nodemanager.keytab" : "/etc/krb5.keytab",
-    "s3.blocksize" : "67108864",
     "mapreduce.task.io.sort.factor" : "10",
     "yarn.nodemanager.disk-health-checker.interval-ms" : "120000",
-    "mapreduce.job.working.dir" : "hdfs://a2115.smile.com:8020/user/jenkins",
+    "mapreduce.job.working.dir" : "hdfs://a2115.smile.com:9820/user/jenkins",
     "yarn.admin.acl" : "*",
     "mapreduce.job.speculative.speculativecap" : "0.1",
     "dfs.namenode.num.checkpoints.retained" : "2",
     "dfs.namenode.delegation.token.renew-interval" : "86400000",
     "yarn.nodemanager.resource.memory-mb" : "8192",
     "io.map.index.interval" : "128",
-    "s3.client-write-packet-size" : "65536",
     "mapreduce.task.files.preserve.failedtasks" : "false",
     "dfs.namenode.http-address" : "a2115.smile.com:20101",
     "ha.zookeeper.session-timeout.ms" : "5000",
     "hadoop.hdfs.configuration.version" : "1",
-    "s3.replication" : "3",
     "dfs.datanode.balance.bandwidthPerSec" : "1048576",
     "mapreduce.reduce.shuffle.connect.timeout" : "180000",
     "hadoop.ssl.enabled" : "false",
@@ -9691,10 +9662,9 @@
     "dfs.namenode.edits.dir" : "${dfs.namenode.name.dir}",
     "ha.health-monitor.sleep-after-disconnect.ms" : "1000",
     "dfs.encrypt.data.transfer" : "false",
-    "dfs.datanode.http.address" : "0.0.0.0:50075",
+    "dfs.datanode.http.address" : "0.0.0.0:9864",
     "mapreduce.terasort.num-rows" : "400000000",
     "mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper",
-    "mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12",
     "dfs.namenode.write.stale.datanode.ratio" : "0.5f",
     "dfs.client.use.datanode.hostname" : "false",
     "yarn.acl.enable" : "true",
@@ -9711,7 +9681,6 @@
     "mapreduce.reduce.log.level" : "INFO",
     "yarn.log-aggregation-enable" : "false",
     "dfs.datanode.sync.behind.writes" : "false",
-    "mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst",
     "dfs.https.server.keystore.resource" : "ssl-server.xml",
     "hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn",
     "dfs.namenode.replication.min" : "1",
@@ -9734,7 +9703,6 @@
     "hadoop.ssl.client.conf" : "ssl-client.xml",
     "dfs.namenode.safemode.threshold-pct" : "0.999f",
     "mapreduce.tasktracker.local.dir.minspacekill" : "0",
-    "mapreduce.jobtracker.retiredjobs.cache.size" : "1000",
     "dfs.blocksize" : "134217728",
     "yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler",
     "mapreduce.job.reduce.slowstart.completedmaps" : "0.8",
@@ -9744,14 +9712,12 @@
     "mapreduce.job.user.name" : "jenkins",
     "mapreduce.tasktracker.outofband.heartbeat" : "false",
     "io.native.lib.available" : "true",
-    "mapreduce.jobtracker.persist.jobstatus.hours" : "0",
     "dfs.client-write-packet-size" : "65536",
     "mapreduce.client.progressmonitor.pollinterval" : "1000",
     "dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name",
     "dfs.ha.log-roll.period" : "120",
     "mapreduce.reduce.input.buffer.percent" : "0.0",
     "mapreduce.map.output.compress.codec" : "org.apache.hadoop.io.compress.SnappyCodec",
-    "mapreduce.map.skip.proc-count.auto-incr" : "true",
     "dfs.client.failover.sleep.base.millis" : "500",
     "dfs.datanode.directoryscan.threads" : "1",
     "mapreduce.jobtracker.address" : "neededForHive:999999",
@@ -9763,13 +9729,12 @@
     "yarn.scheduler.fair.preemption" : "true",
     "mapreduce.reduce.shuffle.parallelcopies" : "5",
     "yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
-    "mapreduce.jobtracker.heartbeats.in.second" : "100",
     "mapreduce.job.maxtaskfailures.per.tracker" : "3",
     "ipc.client.connection.maxidletime" : "10000",
     "mapreduce.shuffle.ssl.enabled" : "false",
     "dfs.namenode.invalidate.work.pct.per.iteration" : "0.32f",
     "dfs.blockreport.intervalMsec" : "21600000",
-    "fs.s3.sleepTimeSeconds" : "10",
+    "fs.s3n.sleepTimeSeconds" : "10",
     "dfs.namenode.replication.considerLoad" : "true",
     "dfs.client.block.write.retries" : "3",
     "hadoop.ssl.server.conf" : "ssl-server.xml",
@@ -9782,8 +9747,6 @@
     "dfs.replication" : "3",
     "ipc.client.tcpnodelay" : "false",
     "dfs.namenode.accesstime.precision" : "3600000",
-    "s3.stream-buffer-size" : "4096",
-    "mapreduce.jobtracker.tasktracker.maxblacklists" : "4",
     "dfs.client.read.shortcircuit.skip.checksum" : "false",
     "mapreduce.job.jvm.numtasks" : "1",
     "mapreduce.task.io.sort.mb" : "100",
@@ -9800,14 +9763,12 @@
     "kfs.stream-buffer-size" : "4096",
     "dfs.ha.tail-edits.period" : "60",
     "hadoop.security.authentication" : "simple",
-    "fs.s3.buffer.dir" : "${hadoop.tmp.dir}/s3",
+    "fs.s3n.buffer.dir" : "${hadoop.tmp.dir}/s3",
     "rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
-    "mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler",
     "yarn.app.mapreduce.am.job.task.listener.thread-count" : "30",
     "dfs.namenode.avoid.read.stale.datanode" : "false",
     "mapreduce.job.reduces" : "0",
     "mapreduce.map.sort.spill.percent" : "0.8",
-    "dfs.client.file-block-storage-locations.timeout" : "60",
     "dfs.datanode.drop.cache.behind.writes" : "false",
     "mapreduce.job.end-notification.retry.interval" : "1",
     "mapreduce.job.maps" : "96",
@@ -9829,7 +9790,6 @@
     "dfs.namenode.replication.interval" : "3",
     "mapreduce.task.skip.start.attempts" : "2",
     "dfs.namenode.https-address" : "a2115.smile.com:20102",
-    "mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo",
     "ipc.client.kill.max" : "10",
     "dfs.ha.automatic-failover.enabled" : "false",
     "mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab",
@@ -9852,11 +9812,9 @@
     "io.mapfile.bloom.size" : "1048576",
     "yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
     "mapreduce.map.maxattempts" : "4",
-    "mapreduce.jobtracker.jobhistory.block.size" : "3145728",
     "yarn.log-aggregation.retain-seconds" : "-1",
     "yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000",
     "ftp.replication" : "3",
-    "mapreduce.jobtracker.http.address" : "0.0.0.0:50030",
     "yarn.nodemanager.health-checker.script.timeout-ms" : "1200000",
     "mapreduce.jobhistory.address" : "a2115.smile.com:10020",
     "mapreduce.jobtracker.taskcache.levels" : "2",
@@ -9870,7 +9828,6 @@
     "dfs.namenode.backup.address" : "0.0.0.0:50100",
     "hadoop.util.hash.type" : "murmur",
     "dfs.block.access.key.update.interval" : "600",
-    "mapreduce.reduce.skip.proc-count.auto-incr" : "true",
     "dfs.datanode.dns.interface" : "default",
     "dfs.datanode.use.datanode.hostname" : "false",
     "mapreduce.job.output.key.class" : "org.apache.hadoop.io.Text",
@@ -9884,7 +9841,7 @@
     "mapreduce.job.dir" : "/user/jenkins/.staging/job_1369942127770_1206",
     "io.map.index.skip" : "0",
     "net.topology.node.switch.mapping.impl" : "org.apache.hadoop.net.ScriptBasedMapping",
-    "fs.s3.maxRetries" : "4",
+    "fs.s3n.maxRetries" : "4",
     "ha.failover-controller.new-active.rpc-timeout.ms" : "60000",
     "s3native.client-write-packet-size" : "65536",
     "yarn.resourcemanager.amliveliness-monitor.interval-ms" : "1000",
@@ -9907,7 +9864,6 @@
     "dfs.datanode.directoryscan.interval" : "21600",
     "yarn.resourcemanager.address" : "a2115.smile.com:8032",
     "yarn.nodemanager.health-checker.interval-ms" : "600000",
-    "dfs.client.file-block-storage-locations.num-threads" : "10",
     "yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs" : "86400",
     "mapreduce.reduce.markreset.buffer.percent" : "0.0",
     "hadoop.security.group.mapping.ldap.directory.search.timeout" : "10000",
@@ -9920,7 +9876,7 @@
     "ftp.stream-buffer-size" : "4096",
     "dfs.namenode.avoid.write.stale.datanode" : "false",
     "hadoop.security.group.mapping.ldap.search.attr.member" : "member",
-    "mapreduce.output.fileoutputformat.outputdir" : "hdfs://a2115.smile.com:8020/user/jenkins/tera-gen-2",
+    "mapreduce.output.fileoutputformat.outputdir" : "hdfs://a2115.smile.com:9820/user/jenkins/tera-gen-2",
     "dfs.blockreport.initialDelay" : "0",
     "yarn.nm.liveness-monitor.expiry-interval-ms" : "600000",
     "hadoop.http.authentication.token.validity" : "36000",
@@ -9946,18 +9902,16 @@
     "yarn.scheduler.maximum-allocation-mb" : "8192",
     "yarn.nodemanager.heartbeat.interval-ms" : "1000",
     "mapreduce.job.userlog.retain.hours" : "24",
-    "dfs.namenode.secondary.http-address" : "0.0.0.0:50090",
+    "dfs.namenode.secondary.http-address" : "0.0.0.0:9868",
     "mapreduce.task.timeout" : "600000",
     "mapreduce.framework.name" : "yarn",
     "ipc.client.idlethreshold" : "4000",
     "ftp.bytes-per-checksum" : "512",
     "ipc.server.tcpnodelay" : "false",
     "dfs.namenode.stale.datanode.interval" : "30000",
-    "s3.bytes-per-checksum" : "512",
     "mapreduce.job.speculative.slowtaskthreshold" : "1.0",
     "yarn.nodemanager.localizer.cache.target-size-mb" : "10240",
     "yarn.nodemanager.remote-app-log-dir" : "/tmp/logs",
-    "fs.s3.block.size" : "67108864",
     "mapreduce.job.queuename" : "sls_queue_1",
     "dfs.client.failover.connection.retries" : "0",
     "hadoop.rpc.protection" : "authentication",
@@ -9966,7 +9920,7 @@
     "hadoop.security.auth_to_local" : "DEFAULT",
     "dfs.secondary.namenode.kerberos.internal.spnego.principal" : "${dfs.web.authentication.kerberos.principal}",
     "ftp.client-write-packet-size" : "65536",
-    "fs.defaultFS" : "hdfs://a2115.smile.com:8020",
+    "fs.defaultFS" : "hdfs://a2115.smile.com:9820",
     "yarn.nodemanager.address" : "0.0.0.0:0",
     "yarn.scheduler.fair.assignmultiple" : "true",
     "yarn.resourcemanager.scheduler.client.thread-count" : "50",
@@ -9980,7 +9934,6 @@
     "mapreduce.reduce.shuffle.read.timeout" : "180000",
     "mapreduce.admin.user.env" : "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native",
     "yarn.app.mapreduce.am.command-opts" : " -Xmx1238932873",
-    "mapreduce.local.clientfactory.class.name" : "org.apache.hadoop.mapred.LocalClientFactory",
     "dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}",
     "fs.permissions.umask-mode" : "022",
     "dfs.client.domain.socket.data.traffic" : "false",
@@ -10011,15 +9964,14 @@
     "fs.s3n.block.size" : "67108864",
     "fs.ftp.host" : "0.0.0.0",
     "hadoop.security.group.mapping" : "org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback",
-    "dfs.datanode.address" : "0.0.0.0:50010",
+    "dfs.datanode.address" : "0.0.0.0:9866",
     "mapreduce.map.skip.maxrecords" : "0",
-    "dfs.datanode.https.address" : "0.0.0.0:50475",
+    "dfs.datanode.https.address" : "0.0.0.0:9865",
     "file.replication" : "1",
     "yarn.resourcemanager.resource-tracker.address" : "a2115.smile.com:8031",
     "dfs.datanode.drop.cache.behind.reads" : "false",
     "hadoop.fuse.connection.timeout" : "300",
     "hadoop.work.around.non.threadsafe.getpwuid" : "false",
-    "mapreduce.jobtracker.restart.recover" : "false",
     "hadoop.tmp.dir" : "/tmp/hadoop-${user.name}",
     "mapreduce.output.fileoutputformat.compress" : "false",
     "mapreduce.tasktracker.indexcache.mb" : "10",
@@ -10029,10 +9981,8 @@
     "hadoop.kerberos.kinit.command" : "kinit",
     "dfs.datanode.du.reserved" : "0",
     "dfs.namenode.fs-limits.max-blocks-per-file" : "1048576",
-    "dfs.webhdfs.enabled" : "false",
     "file.bytes-per-checksum" : "512",
     "mapreduce.task.profile.reduces" : "0-2",
-    "mapreduce.jobtracker.handler.count" : "10",
     "dfs.client.block.write.replace-datanode-on-failure.enable" : "true",
     "mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text",
     "yarn.dispatcher.exit-on-error" : "true",
@@ -10050,7 +10000,6 @@
     "mapreduce.shuffle.ssl.file.buffer.size" : "65536",
     "dfs.permissions.superusergroup" : "supergroup",
     "dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240",
-    "mapreduce.jobtracker.expire.trackers.interval" : "600000",
     "mapreduce.cluster.acls.enabled" : "false",
     "yarn.nodemanager.remote-app-log-dir-suffix" : "logs",
     "ha.failover-controller.graceful-fence.connection.retries" : "1",
@@ -10060,8 +10009,6 @@
     "io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local",
     "dfs.namenode.checkpoint.check.period" : "60",
     "yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000",
-    "mapreduce.jobtracker.maxtasks.perjob" : "-1",
-    "mapreduce.jobtracker.jobhistory.lru.cache.size" : "5",
     "file.blocksize" : "67108864",
     "tfile.io.chunk.size" : "1048576",
     "mapreduce.job.acl-modify-job" : " ",
@@ -10081,7 +10028,7 @@
     "yarn.resourcemanager.resource-tracker.client.thread-count" : "50",
     "mapreduce.tasktracker.dns.nameserver" : "default",
     "mapreduce.map.output.compress" : "true",
-    "dfs.datanode.ipc.address" : "0.0.0.0:50020",
+    "dfs.datanode.ipc.address" : "0.0.0.0:9867",
     "hadoop.ssl.require.client.cert" : "false",
     "yarn.nodemanager.delete.debug-delay-sec" : "0",
     "dfs.datanode.max.transfer.threads" : "4096"
@@ -10212,608 +10159,4 @@
   "clusterReduceMB" : -1,
   "jobMapMB" : 200,
   "jobReduceMB" : 200
-} {
-"priority" : "NORMAL",
-"jobID" : "job_1369942127770_1207",
-"user" : "jenkins",
-"jobName" : "TeraGen",
-"submitTime" : 1371223054499,
-"finishTime" : 1371223153874,
-"queue" : "sls_queue_1",
-"mapTasks" : [ {
-"startTime" : 1371223059053,
-"taskID" : "task_1369942127770_1207_m_000000",
-"taskType" : "MAP",
-"finishTime" : 1371223078206,
-"attempts" : [ ],
-"preferredLocations" : [ ],
-"taskStatus" : "KILLED",
-"inputBytes" : -1,
-"inputRecords" : -1,
-"outputBytes" : -1,
-"outputRecords" : -1
-} ],
-"reduceTasks" : [ ],
-"launchTime" : 1371223058937,
-"totalMaps" : 1,
-"totalReduces" : 0,
-"otherTasks" : [ ],
-"jobProperties" : {
-"mapreduce.job.ubertask.enable" : "false",
-"yarn.resourcemanager.max-completed-applications" : "10000",
-"yarn.resourcemanager.delayed.delegation-token.removal-interval-ms" : "30000",
-"mapreduce.client.submit.file.replication" : "2",
-"yarn.nodemanager.container-manager.thread-count" : "20",
-"mapred.queue.default.acl-administer-jobs" : "*",
-"dfs.image.transfer.bandwidthPerSec" : "0",
-"mapreduce.tasktracker.healthchecker.interval" : "60000",
-"mapreduce.jobtracker.staging.root.dir" : "/user",
-"yarn.resourcemanager.recovery.enabled" : "false",
-"yarn.resourcemanager.am.max-retries" : "1",
-"dfs.block.access.token.lifetime" : "600",
-"fs.AbstractFileSystem.file.impl" : "org.apache.hadoop.fs.local.LocalFs",
-"mapreduce.client.completion.pollinterval" : "5000",
-"mapreduce.job.ubertask.maxreduces" : "1",
-"mapreduce.reduce.shuffle.memory.limit.percent" : "0.25",
-"dfs.domain.socket.path" : "/var/run/hdfs-sockets/dn",
-"hadoop.ssl.keystores.factory.class" : "org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory",
-"hadoop.http.authentication.kerberos.keytab" : "${user.home}/hadoop.keytab",
-"yarn.nodemanager.keytab" : "/etc/krb5.keytab",
-"s3.blocksize" : "67108864",
-"mapreduce.task.io.sort.factor" : "10",
-"yarn.nodemanager.disk-health-checker.interval-ms" : "120000",
-"mapreduce.job.working.dir" : "hdfs://a2115.smile.com:8020/user/jenkins",
-"yarn.admin.acl" : "*",
-"mapreduce.job.speculative.speculativecap" : "0.1",
-"dfs.namenode.num.checkpoints.retained" : "2",
-"dfs.namenode.delegation.token.renew-interval" : "86400000",
-"yarn.nodemanager.resource.memory-mb" : "8192",
-"io.map.index.interval" : "128",
-"s3.client-write-packet-size" : "65536",
-"mapreduce.task.files.preserve.failedtasks" : "false",
-"dfs.namenode.http-address" : "a2115.smile.com:20101",
-"ha.zookeeper.session-timeout.ms" : "5000",
-"hadoop.hdfs.configuration.version" : "1",
-"s3.replication" : "3",
-"dfs.datanode.balance.bandwidthPerSec" : "1048576",
-"mapreduce.reduce.shuffle.connect.timeout" : "180000",
-"hadoop.ssl.enabled" : "false",
-"dfs.journalnode.rpc-address" : "0.0.0.0:8485",
-"yarn.nodemanager.aux-services" : "mapreduce.shuffle",
-"mapreduce.job.counters.max" : "120",
-"dfs.datanode.readahead.bytes" : "4193404",
-"ipc.client.connect.max.retries.on.timeouts" : "45",
-"mapreduce.job.complete.cancel.delegation.tokens" : "true",
-"dfs.client.failover.max.attempts" : "15",
-"dfs.namenode.checkpoint.dir" : "file://${hadoop.tmp.dir}/dfs/namesecondary",
-"dfs.namenode.replication.work.multiplier.per.iteration" : "2",
-"fs.trash.interval" : "1",
-"yarn.resourcemanager.admin.address" : "a2115.smile.com:8033",
-"ha.health-monitor.check-interval.ms" : "1000",
-"mapreduce.job.outputformat.class" : "org.apache.hadoop.examples.terasort.TeraOutputFormat",
-"hadoop.jetty.logs.serve.aliases" : "true",
-"hadoop.http.authentication.kerberos.principal" : "HTTP/_HOST@LOCALHOST",
-"mapreduce.job.reduce.shuffle.consumer.plugin.class" : "org.apache.hadoop.mapreduce.task.reduce.Shuffle",
-"s3native.blocksize" : "67108864",
-"dfs.namenode.edits.dir" : "${dfs.namenode.name.dir}",
-"ha.health-monitor.sleep-after-disconnect.ms" : "1000",
-"dfs.encrypt.data.transfer" : "false",
-"dfs.datanode.http.address" : "0.0.0.0:50075",
-"mapreduce.terasort.num-rows" : "400000000",
-"mapreduce.job.map.class" : "org.apache.hadoop.examples.terasort.TeraGen$SortGenMapper",
-"mapreduce.jobtracker.jobhistory.task.numberprogresssplits" : "12",
-"dfs.namenode.write.stale.datanode.ratio" : "0.5f",
-"dfs.client.use.datanode.hostname" : "false",
-"yarn.acl.enable" : "true",
-"hadoop.security.instrumentation.requires.admin" : "false",
-"yarn.nodemanager.localizer.fetch.thread-count" : "4",
-"hadoop.security.authorization" : "false",
-"user.name" : "jenkins",
-"dfs.namenode.fs-limits.min-block-size" : "1048576",
-"dfs.client.failover.connection.retries.on.timeouts" : "0",
-"hadoop.security.group.mapping.ldap.search.filter.group" : "(objectClass=group)",
-"mapreduce.output.fileoutputformat.compress.codec" : "org.apache.hadoop.io.compress.DefaultCodec",
-"dfs.namenode.safemode.extension" : "30000",
-"mapreduce.shuffle.port" : "8080",
-"mapreduce.reduce.log.level" : "INFO",
-"yarn.log-aggregation-enable" : "false",
-"dfs.datanode.sync.behind.writes" : "false",
-"mapreduce.jobtracker.instrumentation" : "org.apache.hadoop.mapred.JobTrackerMetricsInst",
-"dfs.https.server.keystore.resource" : "ssl-server.xml",
-"hadoop.security.group.mapping.ldap.search.attr.group.name" : "cn",
-"dfs.namenode.replication.min" : "1",
-"mapreduce.map.java.opts" : " -Xmx825955249",
-"yarn.scheduler.fair.allocation.file" : "/etc/yarn/fair-scheduler.xml",
-"s3native.bytes-per-checksum" : "512",
-"mapreduce.tasktracker.tasks.sleeptimebeforesigkill" : "5000",
-"tfile.fs.output.buffer.size" : "262144",
-"yarn.nodemanager.local-dirs" : "${hadoop.tmp.dir}/nm-local-dir",
-"mapreduce.jobtracker.persist.jobstatus.active" : "false",
-"fs.AbstractFileSystem.hdfs.impl" : "org.apache.hadoop.fs.Hdfs",
-"mapreduce.job.map.output.collector.class" : "org.apache.hadoop.mapred.MapTask$MapOutputBuffer",
-"mapreduce.tasktracker.local.dir.minspacestart" : "0",
-"dfs.namenode.safemode.min.datanodes" : "0",
-"hadoop.security.uid.cache.secs" : "14400",
-"dfs.client.https.need-auth" : "false",
-"dfs.client.write.exclude.nodes.cache.expiry.interval.millis" : "600000",
-"dfs.client.https.keystore.resource" : "ssl-client.xml",
-"dfs.namenode.max.objects" : "0",
-"hadoop.ssl.client.conf" : "ssl-client.xml",
-"dfs.namenode.safemode.threshold-pct" : "0.999f",
-"mapreduce.tasktracker.local.dir.minspacekill" : "0",
-"mapreduce.jobtracker.retiredjobs.cache.size" : "1000",
-"dfs.blocksize" : "134217728",
-"yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler",
-"mapreduce.job.reduce.slowstart.completedmaps" : "0.8",
-"mapreduce.job.end-notification.retry.attempts" : "5",
-"mapreduce.job.inputformat.class" : "org.apache.hadoop.examples.terasort.TeraGen$RangeInputFormat",
-"mapreduce.map.memory.mb" : "1024",
-"mapreduce.job.user.name" : "jenkins",
-"mapreduce.tasktracker.outofband.heartbeat" : "false",
-"io.native.lib.available" : "true",
-"mapreduce.jobtracker.persist.jobstatus.hours" : "0",
-"dfs.client-write-packet-size" : "65536",
-"mapreduce.client.progressmonitor.pollinterval" : "1000",
-"dfs.namenode.name.dir" : "file://${hadoop.tmp.dir}/dfs/name",
-"dfs.ha.log-roll.period" : "120",
-"mapreduce.reduce.input.buffer.percent" : "0.0",
-"mapreduce.map.output.compress.codec" : "org.apache.hadoop.io.compress.SnappyCodec",
-"dfs.client.failover.sleep.base.millis" : "500",
-"dfs.datanode.directoryscan.threads" : "1",
-"mapreduce.jobtracker.address" : "neededForHive:999999",
-"mapreduce.cluster.local.dir" : "${hadoop.tmp.dir}/mapred/local",
-"yarn.scheduler.fair.user-as-default-queue" : "true",
-"mapreduce.job.application.attempt.id" : "1",
-"dfs.permissions.enabled" : "true",
-"mapreduce.tasktracker.taskcontroller" : "org.apache.hadoop.mapred.DefaultTaskController",
-"yarn.scheduler.fair.preemption" : "true",
-"mapreduce.reduce.shuffle.parallelcopies" : "5",
-"dfs.support.append" : "true",
-"yarn.nodemanager.env-whitelist" : "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME",
-"mapreduce.jobtracker.heartbeats.in.second" : "100",
-"mapreduce.job.maxtaskfailures.per.tracker" : "3",
-"ipc.client.connection.maxidletime" : "10000",
-"mapreduce.shuffle.ssl.enabled" : "false",
-"dfs.namenode.invalidate.work.pct.per.iteration" : "0.32f",
-"dfs.blockreport.intervalMsec" : "21600000",
-"fs.s3.sleepTimeSeconds" : "10",
-"dfs.namenode.replication.considerLoad" : "true",
-"dfs.client.block.write.retries" : "3",
-"hadoop.ssl.server.conf" : "ssl-server.xml",
-"dfs.namenode.name.dir.restore" : "false",
-"rpc.engine.org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
-"dfs.datanode.hdfs-blocks-metadata.enabled" : "true",
-"ha.zookeeper.parent-znode" : "/hadoop-ha",
-"mapreduce.reduce.merge.inmem.threshold" : "1000",
-"mapreduce.input.fileinputformat.split.minsize" : "0",
-"dfs.replication" : "3",
-"ipc.client.tcpnodelay" : "false",
-"dfs.namenode.accesstime.precision" : "3600000",
-"s3.stream-buffer-size" : "4096",
-"mapreduce.jobtracker.tasktracker.maxblacklists" : "4",
-"dfs.client.read.shortcircuit.skip.checksum" : "false",
-"mapreduce.job.jvm.numtasks" : "1",
-"mapreduce.task.io.sort.mb" : "100",
-"io.file.buffer.size" : "65536",
-"dfs.namenode.audit.loggers" : "default",
-"dfs.namenode.checkpoint.txns" : "1000000",
-"yarn.nodemanager.admin-env" : "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX",
-"mapreduce.job.jar" : "/user/jenkins/.staging/job_1369942127770_1207/job.jar",
-"mapreduce.job.split.metainfo.maxsize" : "10000000",
-"kfs.replication" : "3",
-"rpc.engine.org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
-"yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms" : "1000",
-"mapreduce.reduce.maxattempts" : "4",
-"kfs.stream-buffer-size" : "4096",
-"dfs.ha.tail-edits.period" : "60",
-"hadoop.security.authentication" : "simple",
-"fs.s3.buffer.dir" : "${hadoop.tmp.dir}/s3",
-"rpc.engine.org.apache.hadoop.yarn.api.AMRMProtocolPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
-"mapreduce.jobtracker.taskscheduler" : "org.apache.hadoop.mapred.JobQueueTaskScheduler",
-"yarn.app.mapreduce.am.job.task.listener.thread-count" : "30",
-"dfs.namenode.avoid.read.stale.datanode" : "false",
-"mapreduce.job.reduces" : "0",
-"mapreduce.map.sort.spill.percent" : "0.8",
-"dfs.client.file-block-storage-locations.timeout" : "60",
-"dfs.datanode.drop.cache.behind.writes" : "false",
-"mapreduce.job.end-notification.retry.interval" : "1",
-"mapreduce.job.maps" : "96",
-"mapreduce.job.speculative.slownodethreshold" : "1.0",
-"tfile.fs.input.buffer.size" : "262144",
-"mapreduce.map.speculative" : "false",
-"dfs.block.access.token.enable" : "false",
-"dfs.journalnode.http-address" : "0.0.0.0:8480",
-"mapreduce.job.acl-view-job" : " ",
-"mapreduce.reduce.shuffle.retry-delay.max.ms" : "60000",
-"mapreduce.job.end-notification.max.retry.interval" : "5",
-"ftp.blocksize" : "67108864",
-"mapreduce.tasktracker.http.threads" : "80",
-"mapreduce.reduce.java.opts" : " -Xmx825955249",
-"dfs.datanode.data.dir" : "file://${hadoop.tmp.dir}/dfs/data",
-"ha.failover-controller.cli-check.rpc-timeout.ms" : "20000",
-"dfs.namenode.max.extra.edits.segments.retained" : "10000",
-"dfs.https.port" : "20102",
-"dfs.namenode.replication.interval" : "3",
-"mapreduce.task.skip.start.attempts" : "2",
-"dfs.namenode.https-address" : "a2115.smile.com:20102",
-"mapreduce.jobtracker.persist.jobstatus.dir" : "/jobtracker/jobsInfo",
-"ipc.client.kill.max" : "10",
-"dfs.ha.automatic-failover.enabled" : "false",
-"mapreduce.jobhistory.keytab" : "/etc/security/keytab/jhs.service.keytab",
-"dfs.image.transfer.timeout" : "600000",
-"dfs.client.failover.sleep.max.millis" : "15000",
-"mapreduce.job.end-notification.max.attempts" : "5",
-"mapreduce.task.tmp.dir" : "./tmp",
-"dfs.default.chunk.view.size" : "32768",
-"kfs.bytes-per-checksum" : "512",
-"mapreduce.reduce.memory.mb" : "1024",
-"hadoop.http.filter.initializers" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
-"dfs.datanode.failed.volumes.tolerated" : "0",
-"hadoop.http.authentication.type" : "simple",
-"dfs.datanode.data.dir.perm" : "700",
-"yarn.resourcemanager.client.thread-count" : "50",
-"ipc.server.listen.queue.size" : "128",
-"mapreduce.reduce.skip.maxgroups" : "0",
-"file.stream-buffer-size" : "4096",
-"dfs.namenode.fs-limits.max-directory-items" : "0",
-"io.mapfile.bloom.size" : "1048576",
-"yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
-"mapreduce.map.maxattempts" : "4",
-"mapreduce.jobtracker.jobhistory.block.size" : "3145728",
-"yarn.log-aggregation.retain-seconds" : "-1",
-"yarn.app.mapreduce.am.job.committer.cancel-timeout" : "60000",
-"ftp.replication" : "3",
-"mapreduce.jobtracker.http.address" : "0.0.0.0:50030",
-"yarn.nodemanager.health-checker.script.timeout-ms" : "1200000",
-"mapreduce.jobhistory.address" : "a2115.smile.com:10020",
-"mapreduce.jobtracker.taskcache.levels" : "2",
-"dfs.datanode.dns.nameserver" : "default",
-"mapreduce.application.classpath" : "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*",
-"yarn.nodemanager.log.retain-seconds" : "10800",
-"mapred.child.java.opts" : "-Xmx200m",
-"dfs.replication.max" : "512",
-"map.sort.class" : "org.apache.hadoop.util.QuickSort",
-"dfs.stream-buffer-size" : "4096",
-"dfs.namenode.backup.address" : "0.0.0.0:50100",
-"hadoop.util.hash.type" : "murmur",
-"dfs.block.access.key.update.interval" : "600",
-"dfs.datanode.dns.interface" : "default",
-"dfs.datanode.use.datanode.hostname" : "false",
-"mapreduce.job.output.key.class" : "org.apache.hadoop.io.Text",
-"dfs.client.read.shortcircuit" : "false",
-"dfs.namenode.backup.http-address" : "0.0.0.0:50105",
-"yarn.nodemanager.container-monitor.interval-ms" : "3000",
-"yarn.nodemanager.disk-health-checker.min-healthy-disks" : "0.25",
-"kfs.client-write-packet-size" : "65536",
-"ha.zookeeper.acl" : "world:anyone:rwcda",
-"yarn.nodemanager.sleep-delay-before-sigkill.ms" : "250",
-"mapreduce.job.dir" : "/user/jenkins/.staging/job_1369942127770_1207",
-"io.map.index.skip" : "0",
-"net.topology.node.switch.mapping.impl" : "org.apache.hadoop.net.ScriptBasedMapping",
-"fs.s3.maxRetries" : "4",
-"ha.failover-controller.new-active.rpc-timeout.ms" : "60000",
-"s3native.client-write-packet-size" : "65536",
-"yarn.resourcemanager.amliveliness-monitor.interval-ms" : "1000",
-"hadoop.http.staticuser.user" : "dr.who",
-"mapreduce.reduce.speculative" : "false",
-"mapreduce.client.output.filter" : "FAILED",
-"mapreduce.ifile.readahead.bytes" : "4194304",
-"mapreduce.tasktracker.report.address" : "127.0.0.1:0",
-"mapreduce.task.userlog.limit.kb" : "0",
-"mapreduce.tasktracker.map.tasks.maximum" : "2",
-"hadoop.http.authentication.simple.anonymous.allowed" : "true",
-"hadoop.fuse.timer.period" : "5",
-"dfs.namenode.num.extra.edits.retained" : "1000000",
-"hadoop.rpc.socket.factory.class.default" : "org.apache.hadoop.net.StandardSocketFactory",
-"mapreduce.job.submithostname" : "a2115.smile.com",
-"dfs.namenode.handler.count" : "10",
-"fs.automatic.close" : "false",
-"mapreduce.job.submithostaddress" : "10.20.206.115",
-"mapreduce.tasktracker.healthchecker.script.timeout" : "600000",
-"dfs.datanode.directoryscan.interval" : "21600",
-"yarn.resourcemanager.address" : "a2115.smile.com:8032",
-"yarn.nodemanager.health-checker.interval-ms" : "600000",
-"dfs.client.file-block-storage-locations.num-threads" : "10",
-"yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs" : "86400",
-"mapreduce.reduce.markreset.buffer.percent" : "0.0",
-"hadoop.security.group.mapping.ldap.directory.search.timeout" : "10000",
-"mapreduce.map.log.level" : "INFO",
-"dfs.bytes-per-checksum" : "512",
-"yarn.nodemanager.localizer.address" : "0.0.0.0:8040",
-"dfs.namenode.checkpoint.max-retries" : "3",
-"ha.health-monitor.rpc-timeout.ms" : "45000",
-"yarn.resourcemanager.keytab" : "/etc/krb5.keytab",
-"ftp.stream-buffer-size" : "4096",
-"dfs.namenode.avoid.write.stale.datanode" : "false",
-"hadoop.security.group.mapping.ldap.search.attr.member" : "member",
-"mapreduce.output.fileoutputformat.outputdir" : "hdfs://a2115.smile.com:8020/user/jenkins/tera-gen-1",
-"dfs.blockreport.initialDelay" : "0",
-"yarn.nm.liveness-monitor.expiry-interval-ms" : "600000",
-"hadoop.http.authentication.token.validity" : "36000",
-"dfs.namenode.delegation.token.max-lifetime" : "604800000",
-"mapreduce.job.hdfs-servers" : "${fs.defaultFS}",
-"s3native.replication" : "3",
-"yarn.nodemanager.localizer.client.thread-count" : "5",
-"dfs.heartbeat.interval" : "3",
-"rpc.engine.org.apache.hadoop.ipc.ProtocolMetaInfoPB" : "org.apache.hadoop.ipc.ProtobufRpcEngine",
-"dfs.ha.fencing.ssh.connect-timeout" : "30000",
-"yarn.resourcemanager.container.liveness-monitor.interval-ms" : "600000",
-"yarn.am.liveness-monitor.expiry-interval-ms" : "600000",
-"mapreduce.task.profile" : "false",
-"mapreduce.tasktracker.http.address" : "0.0.0.0:50060",
-"mapreduce.tasktracker.instrumentation" : "org.apache.hadoop.mapred.TaskTrackerMetricsInst",
-"mapreduce.jobhistory.webapp.address" : "a2115.smile.com:19888",
-"ha.failover-controller.graceful-fence.rpc-timeout.ms" : "5000",
-"yarn.ipc.rpc.class" : "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC",
-"mapreduce.job.name" : "TeraGen",
-"kfs.blocksize" : "67108864",
-"yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs" : "86400",
-"mapreduce.job.ubertask.maxmaps" : "9",
-"yarn.scheduler.maximum-allocation-mb" : "8192",
-"yarn.nodemanager.heartbeat.interval-ms" : "1000",
-"mapreduce.job.userlog.retain.hours" : "24",
-"dfs.namenode.secondary.http-address" : "0.0.0.0:50090",
-"mapreduce.task.timeout" : "600000",
-"mapreduce.framework.name" : "yarn",
-"ipc.client.idlethreshold" : "4000",
-"ftp.bytes-per-checksum" : "512",
-"ipc.server.tcpnodelay" : "false",
-"dfs.namenode.stale.datanode.interval" : "30000",
-"s3.bytes-per-checksum" : "512",
-"mapreduce.job.speculative.slowtaskthreshold" : "1.0",
-"yarn.nodemanager.localizer.cache.target-size-mb" : "10240",
-"yarn.nodemanager.remote-app-log-dir" : "/tmp/logs",
-"fs.s3.block.size" : "67108864",
-"mapreduce.job.queuename" : "sls_queue_1",
-"dfs.client.failover.connection.retries" : "0",
-"hadoop.rpc.protection" : "authentication",
-"yarn.scheduler.minimum-allocation-mb" : "1024",
-"yarn.app.mapreduce.client-am.ipc.max-retries" : "1",
-"hadoop.security.auth_to_local" : "DEFAULT",
-"dfs.secondary.namenode.kerberos.internal.spnego.principal" : "${dfs.web.authentication.kerberos.principal}",
-"ftp.client-write-packet-size" : "65536",
-"fs.defaultFS" : "hdfs://a2115.smile.com:8020",
-"yarn.nodemanager.address" : "0.0.0.0:0",
-"yarn.scheduler.fair.assignmultiple" : "true",
-"yarn.resourcemanager.scheduler.client.thread-count" : "50",
-"mapreduce.task.merge.progress.records" : "10000",
-"file.client-write-packet-size" : "65536",
-"yarn.nodemanager.delete.thread-count" : "4",
-"yarn.resourcemanager.scheduler.address" : "a2115.smile.com:8030",
-"fs.trash.checkpoint.interval" : "0",
-"hadoop.http.authentication.signature.secret.file" : "${user.home}/hadoop-http-auth-signature-secret",
-"s3native.stream-buffer-size" : "4096",
-"mapreduce.reduce.shuffle.read.timeout" : "180000",
-"mapreduce.admin.user.env" : "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native",
-"yarn.app.mapreduce.am.command-opts" : " -Xmx1238932873",
-"dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}",
-"fs.permissions.umask-mode" : "022",
-"dfs.client.domain.socket.data.traffic" : "false",
-"hadoop.common.configuration.version" : "0.23.0",
-"mapreduce.tasktracker.dns.interface" : "default",
-"mapreduce.output.fileoutputformat.compress.type" : "BLOCK",
-"mapreduce.ifile.readahead" : "true",
-"hadoop.security.group.mapping.ldap.ssl" : "false",
-"io.serializations" : "org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization",
-"yarn.nodemanager.aux-services.mapreduce.shuffle.class" : "org.apache.hadoop.mapred.ShuffleHandler",
-"fs.df.interval" : "60000",
-"mapreduce.reduce.shuffle.input.buffer.percent" : "0.70",
-"io.seqfile.compress.blocksize" : "1000000",
-"hadoop.security.groups.cache.secs" : "300",
-"ipc.client.connect.max.retries" : "10",
-"dfs.namenode.delegation.key.update-interval" : "86400000",
-"yarn.nodemanager.process-kill-wait.ms" : "2000",
-"yarn.application.classpath" : "$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$YARN_HOME/*,$YARN_HOME/lib/*",
-"yarn.app.mapreduce.client.max-retries" : "3",
-"dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction" : "0.75f",
-"yarn.nodemanager.log-aggregation.compression-type" : "none",
-"hadoop.security.group.mapping.ldap.search.filter.user" : "(&(objectClass=user)(sAMAccountName={0}))",
-"yarn.nodemanager.localizer.cache.cleanup.interval-ms" : "600000",
-"dfs.image.compress" : "false",
-"mapred.mapper.new-api" : "true",
-"yarn.nodemanager.log-dirs" : "${yarn.log.dir}/userlogs",
-"dfs.namenode.kerberos.internal.spnego.principal" : "${dfs.web.authentication.kerberos.principal}",
-"fs.s3n.block.size" : "67108864",
-"fs.ftp.host" : "0.0.0.0",
-"hadoop.security.group.mapping" : "org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback",
-"dfs.datanode.address" : "0.0.0.0:50010",
-"mapreduce.map.skip.maxrecords" : "0",
-"dfs.datanode.https.address" : "0.0.0.0:50475",
-"file.replication" : "1",
-"yarn.resourcemanager.resource-tracker.address" : "a2115.smile.com:8031",
-"dfs.datanode.drop.cache.behind.reads" : "false",
-"hadoop.fuse.connection.timeout" : "300",
-"hadoop.work.around.non.threadsafe.getpwuid" : "false",
-"mapreduce.jobtracker.restart.recover" : "false",
-"hadoop.tmp.dir" : "/tmp/hadoop-${user.name}",
-"mapreduce.output.fileoutputformat.compress" : "false",
-"mapreduce.tasktracker.indexcache.mb" : "10",
-"mapreduce.client.genericoptionsparser.used" : "true",
-"dfs.client.block.write.replace-datanode-on-failure.policy" : "DEFAULT",
-"mapreduce.job.committer.setup.cleanup.needed" : "true",
-"hadoop.kerberos.kinit.command" : "kinit",
-"dfs.datanode.du.reserved" : "0",
-"dfs.namenode.fs-limits.max-blocks-per-file" : "1048576",
-"file.bytes-per-checksum" : "512",
-"mapreduce.task.profile.reduces" : "0-2",
-"mapreduce.jobtracker.handler.count" : "10",
-"dfs.client.block.write.replace-datanode-on-failure.enable" : "true",
-"mapreduce.job.output.value.class" : "org.apache.hadoop.io.Text",
-"yarn.dispatcher.exit-on-error" : "true",
-"net.topology.script.number.args" : "100",
-"mapreduce.task.profile.maps" : "0-2",
-"dfs.namenode.decommission.interval" : "30",
-"dfs.image.compression.codec" : "org.apache.hadoop.io.compress.DefaultCodec",
-"yarn.resourcemanager.webapp.address" : "a2115.smile.com:8088",
-"mapreduce.jobtracker.system.dir" : "${hadoop.tmp.dir}/mapred/system",
-"hadoop.ssl.hostname.verifier" : "DEFAULT",
-"yarn.nodemanager.vmem-pmem-ratio" : "2.1",
-"dfs.namenode.support.allow.format" : "true",
-"mapreduce.jobhistory.principal" : "jhs/_HOST@REALM.TLD",
-"io.mapfile.bloom.error.rate" : "0.005",
-"mapreduce.shuffle.ssl.file.buffer.size" : "65536",
-"dfs.permissions.superusergroup" : "supergroup",
-"dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold" : "10737418240",
-"mapreduce.jobtracker.expire.trackers.interval" : "600000",
-"mapreduce.cluster.acls.enabled" : "false",
-"yarn.nodemanager.remote-app-log-dir-suffix" : "logs",
-"ha.failover-controller.graceful-fence.connection.retries" : "1",
-"ha.health-monitor.connect-retry-interval.ms" : "1000",
-"mapreduce.reduce.shuffle.merge.percent" : "0.66",
-"yarn.app.mapreduce.am.resource.mb" : "1536",
-"io.seqfile.local.dir" : "${hadoop.tmp.dir}/io/local",
-"dfs.namenode.checkpoint.check.period" : "60",
-"yarn.resourcemanager.nm.liveness-monitor.interval-ms" : "1000",
-"mapreduce.jobtracker.maxtasks.perjob" : "-1",
-"mapreduce.jobtracker.jobhistory.lru.cache.size" : "5",
-"file.blocksize" : "67108864",
-"tfile.io.chunk.size" : "1048576",
-"mapreduce.job.acl-modify-job" : " ",
-"yarn.nodemanager.webapp.address" : "0.0.0.0:8042",
-"mapreduce.tasktracker.reduce.tasks.maximum" : "2",
-"io.skip.checksum.errors" : "false",
-"mapreduce.cluster.temp.dir" : "${hadoop.tmp.dir}/mapred/temp",
-"yarn.app.mapreduce.am.staging-dir" : "/user",
-"dfs.namenode.edits.journal-plugin.qjournal" : "org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager",
-"dfs.datanode.handler.count" : "10",
-"fs.ftp.host.port" : "21",
-"dfs.namenode.decommission.nodes.per.interval" : "5",
-"yarn.resourcemanager.admin.client.thread-count" : "1",
-"dfs.namenode.fs-limits.max-component-length" : "0",
-"dfs.namenode.checkpoint.period" : "3600",
-"fs.AbstractFileSystem.viewfs.impl" : "org.apache.hadoop.fs.viewfs.ViewFs",
-"yarn.resourcemanager.resource-tracker.client.thread-count" : "50",
-"mapreduce.tasktracker.dns.nameserver" : "default",
-"mapreduce.map.output.compress" : "true",
-"dfs.datanode.ipc.address" : "0.0.0.0:50020",
-"hadoop.ssl.require.client.cert" : "false",
-"yarn.nodemanager.delete.debug-delay-sec" : "0",
-"dfs.datanode.max.transfer.threads" : "4096"
-},
-"computonsPerMapInputByte" : -1,
-"computonsPerMapOutputByte" : -1,
-"computonsPerReduceInputByte" : -1,
-"computonsPerReduceOutputByte" : -1,
-"heapMegabytes" : 200,
-"outcome" : "SUCCESS",
-"jobtype" : "JAVA",
-"directDependantJobs" : [ ],
-"successfulMapAttemptCDFs" : [ {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-}, {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-}, {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-}, {
-"maximum" : 47021,
-"minimum" : 11143,
-"rankings" : [ {
-"datum" : 13354,
-"relativeRanking" : 0.05
-}, {
-"datum" : 14101,
-"relativeRanking" : 0.1
-}, {
-"datum" : 15609,
-"relativeRanking" : 0.15
-}, {
-"datum" : 15919,
-"relativeRanking" : 0.2
-}, {
-"datum" : 17003,
-"relativeRanking" : 0.25
-}, {
-"datum" : 17109,
-"relativeRanking" : 0.3
-}, {
-"datum" : 18342,
-"relativeRanking" : 0.35
-}, {
-"datum" : 18870,
-"relativeRanking" : 0.4
-}, {
-"datum" : 19127,
-"relativeRanking" : 0.45
-}, {
-"datum" : 19221,
-"relativeRanking" : 0.5
-}, {
-"datum" : 19481,
-"relativeRanking" : 0.55
-}, {
-"datum" : 19896,
-"relativeRanking" : 0.6
-}, {
-"datum" : 20585,
-"relativeRanking" : 0.65
-}, {
-"datum" : 20784,
-"relativeRanking" : 0.7
-}, {
-"datum" : 21452,
-"relativeRanking" : 0.75
-}, {
-"datum" : 21853,
-"relativeRanking" : 0.8
-}, {
-"datum" : 22436,
-"relativeRanking" : 0.85
-}, {
-"datum" : 32646,
-"relativeRanking" : 0.9
-}, {
-"datum" : 41553,
-"relativeRanking" : 0.95
-} ],
-"numberValues" : 96
-} ],
-"failedMapAttemptCDFs" : [ {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-}, {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-}, {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-}, {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-} ],
-"successfulReduceAttemptCDF" : {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-},
-"failedReduceAttemptCDF" : {
-"maximum" : 9223372036854775807,
-"minimum" : -9223372036854775808,
-"rankings" : [ ],
-"numberValues" : 0
-},
-"mapperTriesToSucceed" : [ 1.0 ],
-"failedMapperFraction" : 0.0,
-"relativeTime" : 0,
-"clusterMapMB" : -1,
-"clusterReduceMB" : -1,
-"jobMapMB" : 200,
-"jobReduceMB" : 200
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js b/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
index c5c6482..38f046c 100644
--- a/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
+++ b/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
@@ -1504,7 +1504,7 @@ setDocument = Sizzle.setDocument = function( node ) {
 
 	// Support: IE<10
 	// Check if getElementById returns elements by name
-	// The broken getElementById methods don't pick up programatically-set names,
+	// The broken getElementById methods don't pick up programmatically-set names,
 	// so use a roundabout getElementsByName test
 	support.getById = assert(function( div ) {
 		docElem.appendChild( div ).id = expando;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/ReservationClientUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/ReservationClientUtil.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/ReservationClientUtil.java
new file mode 100644
index 0000000..7c10a57
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/ReservationClientUtil.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.sls;
+
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.records.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Simple support class, used to create reservation requests.
+ */
+public final class ReservationClientUtil {
+
+  private ReservationClientUtil(){
+    //avoid instantiation
+  }
+
+  /**
+   * Creates a request that envelopes a MR jobs, picking max number of maps and
+   * reducers, max durations, and max resources per container.
+   *
+   * @param reservationId the id of the reservation
+   * @param name the name of a reservation
+   * @param maxMapRes maximum resources used by any mapper
+   * @param numberMaps number of mappers
+   * @param maxMapDur maximum duration of any mapper
+   * @param maxRedRes maximum resources used by any reducer
+   * @param numberReduces number of reducers
+   * @param maxRedDur maximum duration of any reducer
+   * @param arrival start time of valid range for reservation
+   * @param deadline deadline for this reservation
+   * @param queueName queue to submit to
+   * @return a submission request
+   */
+  @SuppressWarnings("checkstyle:parameternumber")
+  public static ReservationSubmissionRequest createMRReservation(
+      ReservationId reservationId, String name, Resource maxMapRes,
+      int numberMaps, long maxMapDur, Resource maxRedRes, int numberReduces,
+      long maxRedDur, long arrival, long deadline, String queueName) {
+
+    ReservationRequest mapRR = ReservationRequest.newInstance(maxMapRes,
+        numberMaps, numberMaps, maxMapDur);
+    ReservationRequest redRR = ReservationRequest.newInstance(maxRedRes,
+        numberReduces, numberReduces, maxRedDur);
+
+    List<ReservationRequest> listResReq = new ArrayList<ReservationRequest>();
+    listResReq.add(mapRR);
+    listResReq.add(redRR);
+
+    ReservationRequests reservationRequests = ReservationRequests
+        .newInstance(listResReq, ReservationRequestInterpreter.R_ORDER_NO_GAP);
+    ReservationDefinition resDef = ReservationDefinition.newInstance(arrival,
+        deadline, reservationRequests, name);
+
+    // outermost request
+    ReservationSubmissionRequest request = ReservationSubmissionRequest
+        .newInstance(resDef, queueName, reservationId);
+
+    return request;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d894f910/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
index 0d0745c..76bcb15 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
@@ -34,6 +34,9 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.GnuParser;
@@ -41,9 +44,6 @@ import org.apache.commons.cli.Options;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.sls.utils.SLSUtils;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.ObjectWriter;
 
 @Private
 @Unstable
@@ -127,7 +127,7 @@ public class RumenToSLSConverter {
         ObjectMapper mapper = new ObjectMapper();
         ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
         Iterator<Map> i = mapper.readValues(
-                new JsonFactory().createJsonParser(input), Map.class);
+            new JsonFactory().createParser(input), Map.class);
         while (i.hasNext()) {
           Map m = i.next();
           output.write(writer.writeValueAsString(createSLSJob(m)) + EOL);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message