hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1354832 [3/3] - in /hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/ hadoop-mapreduce-client/hadoop-mapreduce-clie...
Date Thu, 28 Jun 2012 07:00:39 GMT
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
Thu Jun 28 06:59:38 2012
@@ -18,16 +18,27 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.when;
 
 import java.util.HashMap;
-import java.util.Map;
-
 import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -35,10 +46,6 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.QueueACL;
-import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
-
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -289,10 +296,10 @@ public class TestParentQueue {
     conf.setCapacity(Q_B, 50);
     
     final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
-    conf.setCapacity(Q_C, 20);
+    conf.setCapacity(Q_C, 19.5f);
     
     final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D;
-    conf.setCapacity(Q_D, 20);
+    conf.setCapacity(Q_D, 20.5f);
     
     // Define 2-nd level queues
     conf.setQueues(Q_A, new String[] {A1, A2});

Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
Thu Jun 28 06:59:38 2012
@@ -124,11 +124,11 @@ public class TestRMWebServicesCapacitySc
     conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
 
     final String A = CapacitySchedulerConfiguration.ROOT + ".a";
-    conf.setCapacity(A, 10);
+    conf.setCapacity(A, 10.5f);
     conf.setMaximumCapacity(A, 50);
 
     final String B = CapacitySchedulerConfiguration.ROOT + ".b";
-    conf.setCapacity(B, 90);
+    conf.setCapacity(B, 89.5f);
 
     // Define 2nd-level queues
     final String A1 = A + ".a1";

Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
Thu Jun 28 06:59:38 2012
@@ -197,14 +197,14 @@ Hadoop MapReduce Next Generation - Capac
 || Property                            || Description                         |
 *--------------------------------------+--------------------------------------+
 | <<<yarn.scheduler.capacity.<queue-path>.capacity>>> | |
-| | Queue <capacity> in percentage (%). | 
+| | Queue <capacity> in percentage (%) as a float (e.g. 12.5).| 
 | | The sum of capacities for all queues, at each level, must be equal |
 | | to 100. | 
 | | Applications in the queue may consume more resources than the queue's | 
 | | capacity if there are free resources, providing elasticity. |
 *--------------------------------------+--------------------------------------+
 | <<<yarn.scheduler.capacity.<queue-path>.maximum-capacity>>> | 
 | 
-| | Maximum queue capacity in percentage (%). |
+| | Maximum queue capacity in percentage (%) as a float. |
 | | This limits the <elasticity> for applications in the queue. |
 | | Defaults to -1 which disables it. |
 *--------------------------------------+--------------------------------------+

Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm?rev=1354832&r1=1354831&r2=1354832&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
(original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
Thu Jun 28 06:59:38 2012
@@ -712,4 +712,155 @@ digest:hdfs-zkfcs:vlUvLnd8MlacsE80rDuu6O
 
   Even if automatic failover is configured, you may initiate a manual failover
   using the same <<<hdfs haadmin>>> command. It will perform a coordinated
-  failover.
\ No newline at end of file
+  failover.
+
+ 
+* BookKeeper as a Shared storage (EXPERIMENTAL)
+
+   One option for shared storage for the NameNode is BookKeeper. 
+  BookKeeper achieves high availability and strong durability guarantees by replicating
+  edit log entries across multiple storage nodes. The edit log can be striped across 
+  the storage nodes for high performance. Fencing is supported in the protocol, i.e, 
+  BookKeeper will not allow two writers to write the single edit log.
+
+  The meta data for BookKeeper is stored in ZooKeeper.
+  In current HA architecture, a Zookeeper cluster is required for ZKFC. The same cluster
can be
+  for BookKeeper metadata.
+
+  For more details on building a BookKeeper cluster, please refer to the 
+   {{{http://zookeeper.apache.org/bookkeeper/docs/trunk/bookkeeperConfig.html }BookKeeper
documentation}}
+
+ The BookKeeperJournalManager is an implementation of the HDFS JournalManager interface,
which allows custom write ahead logging implementations to be plugged into the HDFS NameNode.
+ 
+ **<<BookKeeper Journal Manager>>
+
+   To use BookKeeperJournalManager, add the following to hdfs-site.xml.
+
+----
+    <property>
+      <name>dfs.namenode.shared.edits.dir</name>
+      <value>bookkeeper://zk1:2181;zk2:2181;zk3:2181/hdfsjournal</value>
+    </property>
+
+    <property>
+      <name>dfs.namenode.edits.journal-plugin.bookkeeper</name>
+      <value>org.apache.hadoop.contrib.bkjournal.BookKeeperJournalManager</value>
+    </property>
+----
+
+   The URI format for bookkeeper is <<<bookkeeper://[zkEnsemble]/[rootZnode]
+   [zookkeeper ensemble]>>> is a list of semi-colon separated, zookeeper host:port
+   pairs. In the example above there are 3 servers, in the ensemble,
+   zk1, zk2 & zk3, each one listening on port 2181.
+
+   <<<[root znode]>>> is the path of the zookeeper znode, under which the
edit log
+   information will be stored.
+
+   The class specified for the journal-plugin must be available in the NameNode's
+   classpath. We explain how to generate a jar file with the journal manager and
+   its dependencies, and how to put it into the classpath below.
+
+ *** <<More configuration options>> 
+
+     * <<dfs.namenode.bookkeeperjournal.output-buffer-size>> - 
+       Number of bytes a bookkeeper journal stream will buffer before
+       forcing a flush. Default is 1024.
+     
+----
+       <property>
+         <name>dfs.namenode.bookkeeperjournal.output-buffer-size</name>
+         <value>1024</value>
+       </property>
+----
+
+     * <<dfs.namenode.bookkeeperjournal.ensemble-size>> - 
+       Number of bookkeeper servers in edit log ensembles. This
+       is the number of bookkeeper servers which need to be available
+       for the edit log to be writable. Default is 3.
+
+----
+       <property>
+         <name>dfs.namenode.bookkeeperjournal.ensemble-size</name>
+         <value>3</value>
+       </property>
+----
+
+     * <<dfs.namenode.bookkeeperjournal.quorum-size>> - 
+       Number of bookkeeper servers in the write quorum. This is the
+       number of bookkeeper servers which must have acknowledged the
+       write of an entry before it is considered written. Default is 2.
+
+----
+       <property>
+         <name>dfs.namenode.bookkeeperjournal.quorum-size</name>
+         <value>2</value>
+       </property>
+----
+
+     * <<dfs.namenode.bookkeeperjournal.digestPw>> - 
+       Password to use when creating edit log segments.
+
+----
+       <property>
+        <name>dfs.namenode.bookkeeperjournal.digestPw</name>
+        <value>myPassword</value>
+       </property>
+----
+
+     * <<dfs.namenode.bookkeeperjournal.zk.session.timeout>> - 
+       Session timeout for Zookeeper client from BookKeeper Journal Manager.
+       Hadoop recommends that this value should be less than the ZKFC 
+       session timeout value. Default value is 3000.
+
+----
+       <property>
+         <name>dfs.namenode.bookkeeperjournal.zk.session.timeout</name>
+         <value>3000</value>
+       </property>
+----
+
+ *** <<Building BookKeeper Journal Manager plugin jar>>
+
+     To generate the distribution packages for BK journal, do the
+     following.
+
+     $ mvn clean package -Pdist
+
+     This will generate a jar with the BookKeeperJournalManager, all the dependencies
+     needed by the journal manager,
+     hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-<VERSION>.jar
+
+     Note that the -Pdist part of the build command is important, as otherwise
+     the dependencies would not be packaged in the jar. The dependencies included in
+     the jar are {{{http://maven.apache.org/plugins/maven-shade-plugin/}shaded}} to
+     avoid conflicts with other dependencies of the NameNode.
+
+ *** <<Putting the BookKeeperJournalManager in the NameNode classpath>>
+
+    To run a HDFS namenode using BookKeeper as a backend, copy the bkjournal
+    jar, generated above, into the lib directory of hdfs. In the standard 
+    distribution of HDFS, this is at $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/
+
+    cp hadoop-hdfs/src/contrib/bkjournal/target/hadoop-hdfs-bkjournal-<VERSION>.jar
$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/
+
+ *** <<Current limitations>> 
+
+      1) NameNode format command will not format the BookKeeper data automatically. 
+         We have to clean the data manually from BookKeeper cluster 
+         and create the /ledgers/available path in Zookeeper. 
+----
+$ zkCli.sh create /ledgers 0
+$ zkCli.sh create /ledgers/available 0
+----
+         Note:
+          bookkeeper://zk1:2181;zk2:2181;zk3:2181/hdfsjournal
+          The final part /hdfsjournal specifies the znode in zookeeper where
+          ledger metadata will be stored. Administrators may set this to anything
+          they wish.
+
+      2) Security in BookKeeper. BookKeeper does not support SASL nor SSL for
+         connections between the NameNode and BookKeeper storage nodes.
+
+      3) Auto-Recovery of storage node failures. Work inprogress 
+      {{{https://issues.apache.org/jira/browse/BOOKKEEPER-237 }BOOKKEEPER-237}}.
+         Currently we have the tools to manually recover the data from failed storage nodes.
\ No newline at end of file

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1346682-1354801

Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1346682-1354801



Mime
View raw message