tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-1349. Add documentation for LocalMode usage. (sseth)
Date Mon, 18 Aug 2014 22:13:34 GMT
Repository: tez
Updated Branches:
  refs/heads/master 7a61e5b85 -> 37ac50aed


TEZ-1349. Add documentation for LocalMode usage. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/37ac50ae
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/37ac50ae
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/37ac50ae

Branch: refs/heads/master
Commit: 37ac50aed43832a0007d514d60afeb341fc8c04e
Parents: 7a61e5b
Author: Siddharth Seth <sseth@apache.org>
Authored: Mon Aug 18 15:13:18 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Mon Aug 18 15:13:18 2014 -0700

----------------------------------------------------------------------
 docs/src/site/apt/localmode.apt                 | 82 ++++++++++++++++++++
 docs/src/site/site.xml                          |  6 +-
 .../org/apache/tez/client/TestLocalMode.java    |  7 ++
 3 files changed, 94 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/37ac50ae/docs/src/site/apt/localmode.apt
----------------------------------------------------------------------
diff --git a/docs/src/site/apt/localmode.apt b/docs/src/site/apt/localmode.apt
new file mode 100644
index 0000000..b38bf77
--- /dev/null
+++ b/docs/src/site/apt/localmode.apt
@@ -0,0 +1,82 @@
+~~ Licensed to the Apache Software Foundation (ASF) under one or more
+~~ contributor license agreements.  See the NOTICE file distributed with
+~~ this work for additional information regarding copyright ownership.
+~~ The ASF licenses this file to You under the Apache License, Version 2.0
+~~ (the "License"); you may not use this file except in compliance with
+~~ the License.  You may obtain a copy of the License at
+~~
+~~     http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License.
+~~
+-----
+  Tez Local Mode
+-----
+
+  Tez LocalMode is a development tool to test Tez jobs without needing to bring up a Hadoop
cluster. LocalMode runs the Tez compoenents - AppMaster, TaskRunner that are used when executing
a job on a cluster. From a developer tool perspective, it offers several advantages.
+
+    * Fast prototyping - Hadoop setup, launch cost etc not involved.
+
+    * Unit testing - Fast execution since the overhead of allocating resources, launching
JVMs etc is removed. 
+
+    * Easy debuggability - Single JVM running all user code.
+
+
+  While majority of the components are re-used in Local Mode, there are some bits which are
not
+
+    * Scheduling and Container Re-Use differs
+
+    * Handling of YARN Local Resources. LocalMode expects necessary jars to be loaded with
the Client when executing.
+
+    * Contains some performance improvements - like skipping RPC invocations since everything
runs within the same JVM.
+
+
+  Running a DAG in LocalMode
+
+    * "tez.local.mode" should be set to true in the confgiuration instance used to create
the TezClient
+
+    * The FileSystem must be configured to the local file system ("fs.defaultFS" must be
set to "file:///"). This is required to be setup in all Configuration instances used to create
a DAG. Typically, when using Tez for testing and prototyping without a Hadoop cluster, this
is not a problem. It becomes a problem when Hadoop Configuration files are in the classpath,
with a different default filesystem configured.
+
+    * Setup the fetchers to make use of local reads instead of fetching from remote nodes.
("tez.runtime.optimize.local.fetch must be set to true)
+
+    * Beyond this, no other changes are required, to make use of LocalMode instead of running
a job on a cluster.
+
+    * If using this in code, the following changes should be made to configuration, after
which this configuration instance becomes the base for all other Configuration instances.
+
++------------------
+Configuration conf = new Configuration();
+conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
+conf.set("fs.defaultFS", "file:///");
+conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
++------------------
+
+    * If using a tez-site.xml config file, it should contain the following entries
+
++------------------
+<property>
+  <name>fs.defaultFS</name>
+  <value>file:///</value>
+</property>
+<property>
+  <name>tez.local.mode<name>
+  <value>true</value>
+</property>
+<property>
+  <name>tez.runtime.optimize.local.fetch</name>
+  <value>true</value>
+</property>
++------------------
+
+
+  Potential pitfalls when moving from LocalMode to a real cluster
+
+    * Resource requirements (CPU, Memory, etc) which would otherwise have been specified
for a YARN Cluster will now start taking affect, and should be considered.
+
+    * The Java Options and Environment variables which may have been setup for the DAG do
not take affect in LocalMode, and could be a source of migration problems.
+
+    * The ObjectRegistry will work within a single task, when running in LocalMode. The behaviour
would be different on a real cluster, where it would work across tasks which share the same
container.
+

http://git-wip-us.apache.org/repos/asf/tez/blob/37ac50ae/docs/src/site/site.xml
----------------------------------------------------------------------
diff --git a/docs/src/site/site.xml b/docs/src/site/site.xml
index 74449c2..637c364 100644
--- a/docs/src/site/site.xml
+++ b/docs/src/site/site.xml
@@ -93,11 +93,15 @@
 
     <menu name="Getting Started">
       <item name="Overview" href="index.html"/>
-      <item name="Install Guide" href="install.html"/>
       <item name="Talks and Meetup Recordings" href="talks.html"/>
       <item name="Project License" href="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
     </menu>
 
+    <menu name="Documentation">
+      <item name="Install Guide" href="install.html"/>
+      <item name="LocalMode" href="localmode.html"/>
+    </menu>
+
     <menu name="Community">
       <item name="Mailing Lists" href="mail-lists.html"/>
       <item name="Issue Tracking" href="issue-tracking.html"/>

http://git-wip-us.apache.org/repos/asf/tez/blob/37ac50ae/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java b/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
index 6e6fa01..5e1b5ba 100644
--- a/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
+++ b/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
@@ -38,6 +38,7 @@ import org.apache.tez.runtime.api.Event;
 import org.apache.tez.runtime.api.LogicalInput;
 import org.apache.tez.runtime.api.LogicalOutput;
 import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
 import org.apache.tez.runtime.library.processor.SleepProcessor;
 import org.junit.Test;
 
@@ -49,6 +50,7 @@ public class TestLocalMode {
     TezConfiguration tezConf1 = new TezConfiguration();
     tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
     tezConf1.set("fs.defaultFS", "file:///");
+    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
     TezClient tezClient1 = new TezClient("commonName", tezConf1, true);
     tezClient1.start();
 
@@ -65,6 +67,7 @@ public class TestLocalMode {
     TezConfiguration tezConf2 = new TezConfiguration();
     tezConf2.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
     tezConf2.set("fs.defaultFS", "file:///");
+    tezConf2.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
     DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
     TezClient tezClient2 = new TezClient("commonName", tezConf2, true);
     tezClient2.start();
@@ -82,6 +85,7 @@ public class TestLocalMode {
     TezConfiguration tezConf1 = new TezConfiguration();
     tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
     tezConf1.set("fs.defaultFS", "file:///");
+    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
     TezClient tezClient1 = new TezClient("commonName", tezConf1, false);
     tezClient1.start();
 
@@ -98,6 +102,7 @@ public class TestLocalMode {
     TezConfiguration tezConf2 = new TezConfiguration();
     tezConf2.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
     tezConf2.set("fs.defaultFS", "file:///");
+    tezConf2.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
     DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
     TezClient tezClient2 = new TezClient("commonName", tezConf2, false);
     tezClient2.start();
@@ -115,6 +120,7 @@ public class TestLocalMode {
     TezConfiguration tezConf1 = new TezConfiguration();
     tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
     tezConf1.set("fs.defaultFS", "file:///");
+    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
     // Run in non-session mode so that the AM terminates
     TezClient tezClient1 = new TezClient("commonName", tezConf1, false);
     tezClient1.start();
@@ -138,6 +144,7 @@ public class TestLocalMode {
     TezConfiguration tezConf1 = new TezConfiguration();
     tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
     tezConf1.set("fs.defaultFS", "file:///");
+    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
     // Run in non-session mode so that the AM terminates
     TezClient tezClient1 = new TezClient("commonName", tezConf1, false);
     tezClient1.start();


Mime
View raw message