zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From build...@apache.org
Subject svn commit: r786254 [10/16] - in /websites/staging/zookeeper/trunk/content: ./ doc/r3.3.3/ doc/r3.3.3/images/ doc/r3.3.3/skin/ doc/r3.3.3/skin/images/
Date Mon, 28 Feb 2011 23:22:00 GMT
Added: websites/staging/zookeeper/trunk/content/doc/r3.3.3/zookeeperInternals.html
==============================================================================
--- websites/staging/zookeeper/trunk/content/doc/r3.3.3/zookeeperInternals.html (added)
+++ websites/staging/zookeeper/trunk/content/doc/r3.3.3/zookeeperInternals.html Mon Feb 28
23:21:57 2011
@@ -0,0 +1,803 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta content="Apache Forrest" name="Generator">
+<meta name="Forrest-version" content="0.8">
+<meta name="Forrest-skin-name" content="pelt">
+<title>ZooKeeper Internals</title>
+<link type="text/css" href="skin/basic.css" rel="stylesheet">
+<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
+<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
+<link type="text/css" href="skin/profile.css" rel="stylesheet">
+<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script
src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script
src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
+<link rel="shortcut icon" href="images/favicon.ico">
+</head>
+<body onload="init()">
+<script type="text/javascript">ndeSetTextSize();</script>
+<div id="top">
+<!--+
+    |breadtrail
+    +-->
+<div class="breadtrail">
+<a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a>
&gt; <a href="http://hadoop.apache.org/zookeeper/">ZooKeeper</a><script
src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
+</div>
+<!--+
+    |header
+    +-->
+<div class="header">
+<!--+
+    |start group logo
+    +-->
+<div class="grouplogo">
+<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg"
title="Apache Hadoop"></a>
+</div>
+<!--+
+    |end group logo
+    +-->
+<!--+
+    |start Project Logo
+    +-->
+<div class="projectlogo">
+<a href="http://hadoop.apache.org/zookeeper/"><img class="logoImage" alt="ZooKeeper"
src="images/zookeeper_small.gif" title="ZooKeeper: distributed coordination"></a>
+</div>
+<!--+
+    |end Project Logo
+    +-->
+<!--+
+    |start Search
+    +-->
+<div class="searchbox">
+<form action="http://www.google.com/search" method="get" class="roundtopsmall">
+<input value="hadoop.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank
(this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search
the site with google">&nbsp; 
+                    <input name="Search" value="Search" type="submit">
+</form>
+</div>
+<!--+
+    |end search
+    +-->
+<!--+
+    |start Tabs
+    +-->
+<ul id="tabs">
+<li>
+<a class="unselected" href="http://hadoop.apache.org/zookeeper/">Project</a>
+</li>
+<li>
+<a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
+</li>
+<li class="current">
+<a class="selected" href="index.html">ZooKeeper 3.3 Documentation</a>
+</li>
+</ul>
+<!--+
+    |end Tabs
+    +-->
+</div>
+</div>
+<div id="main">
+<div id="publishedStrip">
+<!--+
+    |start Subtabs
+    +-->
+<div id="level2tabs"></div>
+<!--+
+    |end Endtabs
+    +-->
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+//  --></script>
+</div>
+<!--+
+    |breadtrail
+    +-->
+<div class="breadtrail">
+
+             &nbsp;
+           </div>
+<!--+
+    |start Menu, mainarea
+    +-->
+<!--+
+    |start Menu
+    +-->
+<div id="menu">
+<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Overview</div>
+<div id="menu_1.1" class="menuitemgroup">
+<div class="menuitem">
+<a href="index.html">Welcome</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperOver.html">Overview</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperStarted.html">Getting Started</a>
+</div>
+<div class="menuitem">
+<a href="releasenotes.html">Release Notes</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Developer</div>
+<div id="menu_1.2" class="menuitemgroup">
+<div class="menuitem">
+<a href="api/index.html">API Docs</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperProgrammers.html">Programmer's Guide</a>
+</div>
+<div class="menuitem">
+<a href="javaExample.html">Java Example</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperTutorial.html">Barrier and Queue Tutorial</a>
+</div>
+<div class="menuitem">
+<a href="recipes.html">Recipes</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">BookKeeper</div>
+<div id="menu_1.3" class="menuitemgroup">
+<div class="menuitem">
+<a href="bookkeeperStarted.html">Getting started</a>
+</div>
+<div class="menuitem">
+<a href="bookkeeperOverview.html">Overview</a>
+</div>
+<div class="menuitem">
+<a href="bookkeeperConfig.html">Setup guide</a>
+</div>
+<div class="menuitem">
+<a href="bookkeeperProgrammer.html">Programmer's guide</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">Admin
&amp; Ops</div>
+<div id="menu_1.4" class="menuitemgroup">
+<div class="menuitem">
+<a href="zookeeperAdmin.html">Administrator's Guide</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperQuotas.html">Quota Guide</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperJMX.html">JMX</a>
+</div>
+<div class="menuitem">
+<a href="zookeeperObservers.html">Observers Guide</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_selected_1.5', 'skin/')" id="menu_selected_1.5Title" class="menutitle"
style="background-image: url('skin/images/chapter_open.gif');">Contributor</div>
+<div id="menu_selected_1.5" class="selectedmenuitemgroup" style="display: block;">
+<div class="menupage">
+<div class="menupagetitle">ZooKeeper Internals</div>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.6', 'skin/')" id="menu_1.6Title" class="menutitle">Miscellaneous</div>
+<div id="menu_1.6" class="menuitemgroup">
+<div class="menuitem">
+<a href="http://wiki.apache.org/hadoop/ZooKeeper">Wiki</a>
+</div>
+<div class="menuitem">
+<a href="http://wiki.apache.org/hadoop/ZooKeeper/FAQ">FAQ</a>
+</div>
+<div class="menuitem">
+<a href="http://hadoop.apache.org/zookeeper/mailing_lists.html">Mailing Lists</a>
+</div>
+</div>
+<div id="credit"></div>
+<div id="roundbottom">
+<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
+<!--+
+  |alternative credits
+  +-->
+<div id="credit2"></div>
+</div>
+<!--+
+    |end Menu
+    +-->
+<!--+
+    |start content
+    +-->
+<div id="content">
+<div title="Portable Document Format" class="pdflink">
+<a class="dida" href="zookeeperInternals.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif"
class="skin"><br>
+        PDF</a>
+</div>
+<h1>ZooKeeper Internals</h1>
+<div id="minitoc-area">
+<ul class="minitoc">
+<li>
+<a href="#ch_Introduction">Introduction</a>
+</li>
+<li>
+<a href="#sc_atomicBroadcast">Atomic Broadcast</a>
+<ul class="minitoc">
+<li>
+<a href="#sc_guaranteesPropertiesDefinitions">Guarantees, Properties, and Definitions</a>
+</li>
+<li>
+<a href="#sc_leaderElection">Leader Activation</a>
+</li>
+<li>
+<a href="#sc_activeMessaging">Active Messaging</a>
+</li>
+<li>
+<a href="#sc_summary">Summary</a>
+</li>
+<li>
+<a href="#sc_comparisons">Comparisons</a>
+</li>
+</ul>
+</li>
+<li>
+<a href="#sc_quorum">Quorums</a>
+</li>
+<li>
+<a href="#sc_logging">Logging</a>
+<ul class="minitoc">
+<li>
+<a href="#sc_developerGuidelines">Developer Guidelines</a>
+<ul class="minitoc">
+<li>
+<a href="#sc_rightLevel">Logging at the Right Level</a>
+</li>
+<li>
+<a href="#sc_log4jIdioms">Use of Standard log4j Idioms</a>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</div>
+  
+
+  
+
+  
+<a name="N10009"></a><a name="ch_Introduction"></a>
+<h2 class="h3">Introduction</h2>
+<div class="section">
+<p>This document contains information on the inner workings of ZooKeeper. 
+    So far, it discusses these topics:
+    </p>
+<ul>    
+
+<li>
+<p>
+<a href="#sc_atomicBroadcast">Atomic Broadcast</a>
+</p>
+</li>
+
+<li>
+<p>
+<a href="#sc_logging">Logging</a>
+</p>
+</li>
+
+</ul>
+</div>
+
+
+<a name="N10022"></a><a name="sc_atomicBroadcast"></a>
+<h2 class="h3">Atomic Broadcast</h2>
+<div class="section">
+<p>
+At the heart of ZooKeeper is an atomic messaging system that keeps all of the servers in
sync.</p>
+<a name="N1002B"></a><a name="sc_guaranteesPropertiesDefinitions"></a>
+<h3 class="h4">Guarantees, Properties, and Definitions</h3>
+<p>
+The specific guarantees provided by the messaging system used by ZooKeeper are the following:</p>
+<dl>
+
+
+<dt>
+<term>
+<em>Reliable delivery</em>
+</term>
+</dt>
+<dd>
+<p>If a message, m, is delivered 
+by one server, it will be eventually delivered by all servers.</p>
+</dd>
+
+
+<dt>
+<term>
+<em>Total order</em>
+</term>
+</dt>
+<dd>
+<p> If a message is 
+delivered before message b by one server, a will be delivered before b by all 
+servers. If a and b are delivered messages, either a will be delivered before b 
+or b will be delivered before a.</p>
+</dd>
+
+
+<dt>
+<term>
+<em>Causal order</em> 
+</term>
+</dt>
+<dd>
+<p>
+If a message b is sent after a message a has been delivered by the sender of b, 
+a must be ordered before b. If a sender sends c after sending b, c must be ordered after
b.
+</p>
+</dd>
+
+
+</dl>
+<p>
+The ZooKeeper messaging system also needs to be efficient, reliable, and easy to 
+implement and maintain. We make heavy use of messaging, so we need the system to 
+be able to handle thousands of requests per second. Although we can require at 
+least k+1 correct servers to send new messages, we must be able to recover from 
+correlated failures such as power outages. When we implemented the system we had 
+little time and few engineering resources, so we needed a protocol that is 
+accessible to engineers and is easy to implement. We found that our protocol 
+satisfied all of these goals.
+
+</p>
+<p>
+Our protocol assumes that we can construct point-to-point FIFO channels between 
+the servers. While similar services usually assume message delivery that can 
+lose or reorder messages, our assumption of FIFO channels is very practical 
+given that we use TCP for communication. Specifically we rely on the following property of
TCP:</p>
+<dl>
+
+
+<dt>
+<term>
+<em>Ordered delivery</em>
+</term>
+</dt>
+<dd>
+<p>Data is delivered in the same order it is sent and a message m is 
+delivered only after all messages sent before m have been delivered. 
+(The corollary to this is that if message m is lost all messages after m will be lost.)</p>
+</dd>
+
+
+<dt>
+<term>
+<em>No message after close</em>
+</term>
+</dt>
+<dd>
+<p>Once a FIFO channel is closed, no messages will be received from it.</p>
+</dd>
+
+
+</dl>
+<p>
+FLP proved that consensus cannot be achieved in asynchronous distributed systems 
+if failures are possible. To ensure we achieve consensus in the presence of failures 
+we use timeouts. However, we rely on times for liveness not for correctness. So, 
+if timeouts stop working (clocks malfunction for example) the messaging system may 
+hang, but it will not violate its guarantees.</p>
+<p>When describing the ZooKeeper messaging protocol we will talk of packets, 
+proposals, and messages:</p>
+<dl>
+
+<dt>
+<term>
+<em>Packet</em>
+</term>
+</dt>
+<dd>
+<p>a sequence of bytes sent through a FIFO channel</p>
+</dd>
+<dt>
+<term>
+<em>Proposal</em>
+</term>
+</dt>
+<dd>
+<p>a unit of agreement. Proposals are agreed upon by exchanging packets 
+with a quorum of ZooKeeper servers. Most proposals contain messages, however the 
+NEW_LEADER proposal is an example of a proposal that does not correspond to a message.</p>
+</dd>
+<dt>
+<term>
+<em>Message</em>
+</term>
+</dt>
+<dd>
+<p>a sequence of bytes to be atomically broadcast to all ZooKeeper 
+servers. A message put into a proposal and agreed upon before it is delivered.</p>
+</dd>
+
+
+</dl>
+<p>
+As stated above, ZooKeeper guarantees a total order of messages, and it also 
+guarantees a total order of proposals. ZooKeeper exposes the total ordering using
+a ZooKeeper transaction id (<em>zxid</em>). All proposals will be stamped with
a zxid when 
+it is proposed and exactly reflects the total ordering. Proposals are sent to all 
+ZooKeeper servers and committed when a quorum of them acknowledge the proposal. 
+If a proposal contains a message, the message will be delivered when the proposal 
+is committed. Acknowledgement means the server has recorded the proposal to persistent storage.

+Our quorums have the requirement that any pair of quorum must have at least one server 
+in common. We ensure this by requiring that all quorums have size (<em>n/2+1</em>)
where 
+n is the number of servers that make up a ZooKeeper service.
+</p>
+<p>
+The zxid has two parts: the epoch and a counter. In our implementation the zxid 
+is a 64-bit number. We use the high order 32-bits for the epoch and the low order 
+32-bits for the counter. Because it has two parts represent the zxid both as a 
+number and as a pair of integers, (<em>epoch, count</em>). The epoch number represents
a 
+change in leadership. Each time a new leader comes into power it will have its 
+own epoch number. We have a simple algorithm to assign a unique zxid to a proposal: 
+the leader simply increments the zxid to obtain a unique zxid for each proposal. 
+<em>Leadership activation will ensure that only one leader uses a given epoch, so our

+simple algorithm guarantees that every proposal will have a unique id.</em>
+
+</p>
+<p>
+ZooKeeper messaging consists of two phases:</p>
+<dl>
+
+<dt>
+<term>
+<em>Leader activation</em>
+</term>
+</dt>
+<dd>
+<p>In this phase a leader establishes the correct state of the system 
+and gets ready to start making proposals.</p>
+</dd>
+
+
+<dt>
+<term>
+<em>Active messaging</em>
+</term>
+</dt>
+<dd>
+<p>In this phase a leader accepts messages to propose and coordinates message delivery.</p>
+</dd>
+
+</dl>
+<p>
+ZooKeeper is a holistic protocol. We do not focus on individual proposals, rather 
+look at the stream of proposals as a whole. Our strict ordering allows us to do this 
+efficiently and greatly simplifies our protocol. Leadership activation embodies 
+this holistic concept. A leader becomes active only when a quorum of followers 
+(The leader counts as a follower as well. You can always vote for yourself ) has synced 
+up with the leader, they have the same state. This state consists of all of the 
+proposals that the leader believes have been committed and the proposal to follow 
+the leader, the NEW_LEADER proposal. (Hopefully you are thinking to 
+yourself, <em>Does the set of proposals that the leader believes has been committed

+included all the proposals that really have been committed?</em> The answer is <em>yes</em>.

+Below, we make clear why.)
+</p>
+<a name="N100B9"></a><a name="sc_leaderElection"></a>
+<h3 class="h4">Leader Activation</h3>
+<p>
+Leader activation includes leader election. We currently have two leader election 
+algorithms in ZooKeeper: LeaderElection and FastLeaderElection (AuthFastLeaderElection 
+is a variant of FastLeaderElection that uses UDP and allows servers to perform a simple
+form of authentication to avoid IP spoofing). ZooKeeper messaging doesn't care about the

+exact method of electing a leader has long as the following holds:
+</p>
+<ul>
+
+
+<li>
+<p>The leader has seen the highest zxid of all the followers.</p>
+</li>
+
+<li>
+<p>A quorum of servers have committed to following the leader.</p>
+</li>
+
+
+</ul>
+<p>
+Of these two requirements only the first, the highest zxid amoung the followers 
+needs to hold for correct operation. The second requirement, a quorum of followers, 
+just needs to hold with high probability. We are going to recheck the second requirement,

+so if a failure happens during or after the leader election and quorum is lost, 
+we will recover by abandoning leader activation and running another election.
+</p>
+<p>
+After leader election a single server will be designated as a leader and start 
+waiting for followers to connect. The rest of the servers will try to connect to 
+the leader. The leader will sync up with followers by sending any proposals they 
+are missing, or if a follower is missing too many proposals, it will send a full 
+snapshot of the state to the follower.
+</p>
+<p>
+There is a corner case in which a follower that has proposals, U, not seen 
+by a leader arrives. Proposals are seen in order, so the proposals of U will have a zxids

+higher than zxids seen by the leader. The follower must have arrived after the 
+leader election, otherwise the follower would have been elected leader given that 
+it has seen a higher zxid. Since committed proposals must be seen by a quorum of 
+servers, and a quorum of servers that elected the leader did not see U, the proposals 
+of you have not been committed, so they can be discarded. When the follower connects 
+to the leader, the leader will tell the follower to discard U.
+</p>
+<p>
+A new leader establishes a zxid to start using for new proposals by getting the 
+epoch, e, of the highest zxid it has seen and setting the next zxid to use to be 
+(e+1, 0), fter the leader syncs with a follower, it will propose a NEW_LEADER 
+proposal. Once the NEW_LEADER proposal has been committed, the leader will activate 
+and start receiving and issuing proposals.
+</p>
+<p>
+It all sounds complicated but here are the basic rules of operation during leader 
+activation:
+</p>
+<ul>
+
+<li>
+<p>A follower will ACK the NEW_LEADER proposal after it has synced with the leader.</p>
+</li>
+
+<li>
+<p>A follower will only ACK a NEW_LEADER proposal with a given zxid from a single server.</p>
+</li>
+
+<li>
+<p>A new leader will COMMIT the NEW_LEADER proposal when a quorum of followers have
ACKed it.</p>
+</li>
+
+<li>
+<p>A follower will commit any state it received from the leader when the NEW_LEADER
proposal is COMMIT.</p>
+</li>
+
+<li>
+<p>A new leader will not accept new proposals until the NEW_LEADER proposal has been
COMMITED.</p>
+</li>
+
+</ul>
+<p>
+If leader election terminates erroneously, we don't have a problem since the 
+NEW_LEADER proposal will not be committed since the leader will not have quorum. 
+When this happens, the leader and any remaining followers will timeout and go back 
+to leader election.
+</p>
+<a name="N100F7"></a><a name="sc_activeMessaging"></a>
+<h3 class="h4">Active Messaging</h3>
+<p>
+Leader Activation does all the heavy lifting. Once the leader is coronated he can 
+start blasting out proposals. As long as he remains the leader no other leader can 
+emerge since no other leader will be able to get a quorum of followers. If a new 
+leader does emerge, 
+it means that the leader has lost quorum, and the new leader will clean up any 
+mess left over during her leadership activation.
+</p>
+<p>ZooKeeper messaging operates similar to a classic two-phase commit.</p>
+<img alt="" src="images/2pc.jpg"><p>
+All communication channels are FIFO, so everything is done in order. Specifically 
+the following operating constraints are observed:</p>
+<ul>
+
+
+<li>
+<p>The leader sends proposals to all followers using 
+the same order. Moreover, this order follows the order in which requests have been 
+received. Because we use FIFO channels this means that followers also receive proposals in
order.
+</p>
+</li>
+
+
+<li>
+<p>Followers process messages in the order they are received. This 
+means that messages will be ACKed in order and the leader will receive ACKs from 
+followers in order, due to the FIFO channels. It also means that if message $m$ 
+has been written to non-volatile storage, all messages that were proposed before 
+$m$ have been written to non-volatile storage.</p>
+</li>
+
+
+<li>
+<p>The leader will issue a COMMIT to all followers as soon as a 
+quorum of followers have ACKed a message. Since messages are ACKed in order, 
+COMMITs will be sent by the leader as received by the followers in order.</p>
+</li>
+
+
+<li>
+<p>COMMITs are processed in order. Followers deliver a proposals 
+message when that proposal is committed.</p>
+</li>
+
+
+</ul>
+<a name="N1011E"></a><a name="sc_summary"></a>
+<h3 class="h4">Summary</h3>
+<p>So there you go. Why does it work? Specifically, why does is set of proposals 
+believed by a new leader always contain any proposal that has actually been committed? 
+First, all proposals have a unique zxid, so unlike other protocols, we never have 
+to worry about two different values being proposed for the same zxid; followers 
+(a leader is also a follower) see and record proposals in order; proposals are 
+committed in order; there is only one active leader at a time since followers only 
+follow a single leader at a time; a new leader has seen all committed proposals 
+from the previous epoch since it has seen the highest zxid from a quorum of servers; 
+any uncommited proposals from a previous epoch seen by a new leader will be committed 
+by that leader before it becomes active.</p>
+<a name="N10127"></a><a name="sc_comparisons"></a>
+<h3 class="h4">Comparisons</h3>
+<p>
+Isn't this just Multi-Paxos? No, Multi-Paxos requires some way of assuring that 
+there is only a single coordinator. We do not count on such assurances. Instead 
+we use the leader activation to recover from leadership change or old leaders 
+believing they are still active.
+</p>
+<p>
+Isn't this just Paxos? Your active messaging phase looks just like phase 2 of Paxos? 
+Actually, to us active messaging looks just like 2 phase commit without the need to 
+handle aborts. Active messaging is different from both in the sense that it has 
+cross proposal ordering requirements. If we do not maintain strict FIFO ordering of 
+all packets, it all falls apart. Also, our leader activation phase is different from 
+both of them. In particular, our use of epochs allows us to skip blocks of uncommitted
+proposals and to not worry about duplicate proposals for a given zxid.
+</p>
+</div>
+
+
+<a name="N10134"></a><a name="sc_quorum"></a>
+<h2 class="h3">Quorums</h2>
+<div class="section">
+<p>
+Atomic broadcast and leader election use the notion of quorum to guarantee a consistent
+view of the system. By default, ZooKeeper uses majority quorums, which means that every
+voting that happens in one of these protocols requires a majority to vote on. One example
is
+acknowledging a leader proposal: the leader can only commit once it receives an
+acknowledgement from a quorum of servers.
+</p>
+<p>
+If we extract the properties that we really need from our use of majorities, we have that
we only
+need to guarantee that groups of processes used to validate an operation by voting (e.g.,
acknowledging
+a leader proposal) pairwise intersect in at least one server. Using majorities guarantees
such a property.
+However, there are other ways of constructing quorums different from majorities. For example,
we can assign
+weights to the votes of servers, and say that the votes of some servers are more important.
To obtain a quorum,
+we get enough votes so that the sum of weights of all votes is larger than half of the total
sum of all weights.    
+</p>
+<p>
+A different construction that uses weights and is useful in wide-area deployments (co-locations)
is a hierarchical
+one. With this construction, we split the servers into disjoint groups and assign weights
to processes. To form 
+a quorum, we have to get a hold of enough servers from a majority of groups G, such that
for each group g in G,
+the sum of votes from g is larger than half of the sum of weights in g. Interestingly, this
construction enables
+smaller quorums. If we have, for example, 9 servers, we split them into 3 groups, and assign
a weight of 1 to each
+server, then we are able to form quorums of size 4. Note that two subsets of processes composed
each of a majority
+of servers from each of a majority of groups necessarily have a non-empty intersection. It
is reasonable to expect
+that a majority of co-locations will have a majority of servers available with high probability.

+</p>
+<p>
+With ZooKeeper, we provide a user with the ability of configuring servers to use majority
quorums, weights, or a 
+hierarchy of groups.
+</p>
+</div>
+
+
+<a name="N10147"></a><a name="sc_logging"></a>
+<h2 class="h3">Logging</h2>
+<div class="section">
+<p>
+ZooKeeper uses 
+<a href="http://logging.apache.org/log4j">log4j</a>
+version 1.2 as its logging infrastructure. For information on configuring log4j for
+ZooKeeper, see the <a href="zookeeperAdmin.html#sc_logging">Logging</a> section

+of the <a href="zookeeperAdmin.html">ZooKeeper Administrator's Guide.</a>
+
+</p>
+<a name="N1015C"></a><a name="sc_developerGuidelines"></a>
+<h3 class="h4">Developer Guidelines</h3>
+<p>Please follow these guidelines when submitting code. Patch reviewers will look for
the following:</p>
+<a name="N10164"></a><a name="sc_rightLevel"></a>
+<h4>Logging at the Right Level</h4>
+<p>
+There are <a href="http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/Level.html#FATAL">6
levels of logging in log4j</a>. 
+It's important to pick the right one. In order of higher to lower severity:</p>
+<ol>
+   
+<li>
+<p> FATAL level designates very severe error events that will presumably lead the application
to abort</p>
+</li>
+   
+<li>
+<p>ERROR level designates error events that might still allow the application to continue
running.</p>
+</li>
+   
+<li>
+<p>WARN level designates potentially harmful situations.</p>
+</li>
+   
+<li>
+<p>INFO level designates informational messages that highlight the progress of the
application at coarse-grained level.</p>
+</li>
+   
+<li>
+<p>EBUG Level designates fine-grained informational events that are most useful to
debug an application.</p>
+</li>
+   
+<li>
+<p>TRACE Level designates finer-grained informational events than the DEBUG.</p>
+</li>
+
+</ol>
+<p>
+ZooKeeper is typically run in production such that log messages of INFO level 
+severity and higher (more severe) are output to the log.</p>
+<a name="N1018F"></a><a name="sc_log4jIdioms"></a>
+<h4>Use of Standard log4j Idioms</h4>
+<p>
+<em>Static Message Logging</em>
+</p>
+<pre class="code">
+LOG.debug("process completed successfully!");
+</pre>
+<p>However when creating a message from a number of components (string 
+concatenation), the log call should be wrapped with a "isXEnabled()" call. this 
+eliminates the string concatenation overhead when debug level logging is not enabled.
+</p>
+<pre class="code">
+if (LOG.isDebugEnabled()) {
+    LOG.debug("got " + count + " messages in " + time + " minutes");
+}
+</pre>
+<p>
+<em>Naming</em>
+</p>
+<p>
+Loggers should be named after the class in which they are used. (See the 
+<a href="http://logging.apache.org/log4j/1.2/faq.html#2.4">log4j faq</a> 
+for reasons why this is a good idea.)
+</p>
+<pre class="code">
+public class Foo {
+    private static final Logger LOG = Logger.getLogger(Foo.class);
+    ....
+    public Foo() {
+       LOG.info("constructing Foo");
+</pre>
+<p>
+<em>Exception handling</em>
+</p>
+<pre class="code">
+try {
+  // code
+} catch (XYZException e) {
+  // do this
+  LOG.error("Something bad happened", e);
+  // don't do this (generally)
+  // LOG.error(e);
+  // why? because "don't do" case hides the stack trace
+ 
+  // continue process here as you need... recover or (re)throw
+}
+</pre>
+</div>
+
+
+<p align="right">
+<font size="-2"></font>
+</p>
+</div>
+<!--+
+    |end content
+    +-->
+<div class="clearboth">&nbsp;</div>
+</div>
+<div id="footer">
+<!--+
+    |start bottomstrip
+    +-->
+<div class="lastmodified">
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+//  --></script>
+</div>
+<div class="copyright">
+        Copyright &copy;
+         2008 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
+</div>
+<!--+
+    |end bottomstrip
+    +-->
+</div>
+</body>
+</html>

Added: websites/staging/zookeeper/trunk/content/doc/r3.3.3/zookeeperInternals.pdf
==============================================================================
--- websites/staging/zookeeper/trunk/content/doc/r3.3.3/zookeeperInternals.pdf (added)
+++ websites/staging/zookeeper/trunk/content/doc/r3.3.3/zookeeperInternals.pdf Mon Feb 28
23:21:57 2011
@@ -0,0 +1,783 @@
+%PDF-1.3
+%ª«¬­
+4 0 obj
+<< /Type /Info
+/Producer (FOP 0.20.5) >>
+endobj
+5 0 obj
+<< /Length 753 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+Gaua=995Pr&BF6e$6Kq\FoCB@9Y0%[gTAuUEpD]C"Xf323&QdIhnCuoZ)QH=8uk8g@Ecr_T:,N0lSNj-9*&H4JTMPJqj.HlC]b*-q-fB;Z&g,"_\f0apq$6pe41s3;D5k%'#j#'idKWT?&Q!kK'si\d-")WPbU?'UbJUb4AUa=G6BVsP<7e9@6D&tii//c8o>_&eAg<Wb045%DKZLD)X#"ofRsApb4q&0qsNN(pHF&eDA]e(]<!S`]`e;HE2Lg?`tVEuKj[3DYn;]b`'L9*IJ6aBWZlM"^",g:R:t<"f"f!j4J9#0FBb1bAhZ6?_i*EB:rpZXGFd*te<XoW\J=n@Ppu^@(_)?`GK,Wt'QmglV0V$"136jKiUutS#U>+Sn_4TLI_`W2q9\7+f?+g-?Hggt4ZGNSZY=@o6ghbS1q4XSK6h"u8N)R\(A2hlDL1=)e(bWtaWcg@F8H!!>u_2ZQs\](4nj1\c"tIk]Kna+lj(1'eP>>&H22(N2e'YLMB5JA3_cPOWJeaOa,f!t3i0H'51hUa;9c8^Njr#cMmY]UqsO2n4%fhYKcAM$2?jreA;*r'3H8M0lsc`?)nD<3gA+?if+khIMd$,+1)Og%M>CKrr9-qHrK.Vc#Y([Yqq"5tE1eN0":r?&SlEgWI1V?orNu!cCA5[mjM7!@O%ugj46U/Ngq&^72o;i+#XYf&dV%g'eP9YhrMVXA:SGIa>Lh8L^DfKmX=TpiceWiSCo#q2*'3_!KY:Os~>
+endstream
+endobj
+6 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 5 0 R
+/Annots 7 0 R
+>>
+endobj
+7 0 obj
+[
+8 0 R
+10 0 R
+12 0 R
+14 0 R
+16 0 R
+18 0 R
+20 0 R
+22 0 R
+24 0 R
+26 0 R
+]
+endobj
+8 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 102.0 529.541 169.328 517.541 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 9 0 R
+/H /I
+>>
+endobj
+10 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 102.0 511.341 196.988 499.341 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 11 0 R
+/H /I
+>>
+endobj
+12 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 493.141 312.464 481.141 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 13 0 R
+/H /I
+>>
+endobj
+14 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 474.941 210.476 462.941 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 15 0 R
+/H /I
+>>
+endobj
+16 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 456.741 210.488 444.741 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 17 0 R
+/H /I
+>>
+endobj
+18 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 438.541 170.168 426.541 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 19 0 R
+/H /I
+>>
+endobj
+20 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 420.341 186.836 408.341 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 21 0 R
+/H /I
+>>
+endobj
+22 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 102.0 402.141 154.664 390.141 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 23 0 R
+/H /I
+>>
+endobj
+24 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 102.0 383.941 150.668 371.941 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 25 0 R
+/H /I
+>>
+endobj
+26 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 365.741 228.476 353.741 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 27 0 R
+/H /I
+>>
+endobj
+28 0 obj
+<< /Length 2020 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+Gau0DgQ(#H&:O:SkcmF[N'uM!QDE=ZD1SL>JH0Q`^m-@<=@mU*C(:2k8(MN*93ro0&R]$&i`sdQ*7"&uG4Xu_G.`@8c8ORic@4e60)Y=2e:kRan;6<ucF.ioq8jj7S3%WIq`hctN0:e61I>+qS:@*sn!i4CRr<8WOmm8;T(W8L7$/-fcJ6CCEYN^=2LfT#1l=Z@UTJ!0.J64C[lUu'Tgu4HRFNUebd`kYatQ\dDa]j`<O12[IM;`>#EHR%\/L\]W?+L98)e&Crr--E5<(lJKn/qLa58mU09KpYk"b-Aesbn7/)+Di+gghWN75j1,EF%t^N?A[h0\aLL%%#.>7"Xc%<4eic^[f"S;dc2i8Z(/8aMZQojD#?N_>NhG#$?GlgT!OeD;)8G-Y#,\=MlfN@[N,U`/Q6<6$ps?QY<1_eDBa,[!5nR+NHI-GE??m4tQ`%%p\4+3$JN<eD;h'"+;W,`7b6is/ureOaEFT5q]%PT26g;6?N6'C+$>*(<HQLSeESEO[XWO<u7GBj\H&l]X=#f2Q\PKd?"6j0`$g])u]g4B#giPYaY$<6B%:=37j^/R&+=5&8&/C^!1P+$gL)>nllg=gi+KT")'SB<J='QP.%<6D(<;i#);HcioLe5nN\Y_gM?aAr9TCF,_%!UQPV@&_8;0G2qR`Sr62t>c8H+*CKqTD;'d!pJ\@Icp7qkQ2!L<@t9S%'0-QHIP?9,4"C%OFrXZ^E*beW79jWA,\\TtR,ScRfAR+1R$-=>>S`#UJH6u1#?Rk2[2Vr/d&9&#M&1b,'GkV+Ek)Ka#I<MTaLOdLP63,op%mR@>\$c<3/W1\EkZ9L[7_@Yl70ObDR,JA2=c9XnL(i7CFGnJC:.*UF_+"57G(Cd@=!U9d]d26=/oO>k\%-mUP&d9Y[%beLVDduB@4`:O$M%B+MYd^M/k-&4&!MQclSC)^^T&V4Cf6#+H9F4bo#2$Kis?B-qj6JiZQe<Ft8"OTR`&MpRQ'
 PMQE[?_gPS#A[VMg[!#s=IrAL8f5ZO\X3G7e=bt(_4S4+.VD0@/\rGAUqh$=ZX;tZl&+d2,f%Sb:1)!IkIH82J(PH^=cL^ir27-L2"L:HYRh)XU%D.n]EbeA2M>7eo%qP*gUX3tUZWHqGV7GRuAq`_7:rPG<hoL)lGn-s.J94.BX7"hLl>&B.kBY1D4;PT#&SM"N;\BT\LBWtt6oKZC$M#!,k3UK5?D!>U)Gb0/3=!Ka)]54]\%u7js(MkQ!+au#>^pkr?]3SiLB\=sHUQ(D!ps)MMqq8.>KsRl/Y&BM0T>Jie58%(pP4$L1Yg&1U4l%'.*i:HE8!PUlldfU.@qDn`M>M4E1$1MpID6"aq%#%FRs7a:+R4BE\P:8q1O=BPq4:((!PTXaI+GkKpu^\B*_S_ZU]9ZXpWopBM:q-X%hUi:A4kQBC+\U;QDSfT0=rNWY!0P\;1c-L;T,50^oFlLi[.4%fdehQ!Jttj/s>\gifa`/Cm3K.VWE?$g]Xo8o#cu^^;Y/6f+<9(OX3-<`%NP8`d6qKO.%)V>a#4[MJr%/XJ=N6Kc4;kf/R?M/Y-,2OM1#4LfM<cM;G6!2amEil&=N?R->Y,&L;=RGfQFipVi=]DNN?n)*#P5R2ij.pc>;M`FqRbJM'Y_<$7sA_Jqi3=),r>*Pah(4E21O^V=,4di0N0WrK2g_bgiH+5D]$>J=#a:uU5^M,b9pVfBA1ZW62e&C2(j<Z]kO$#$&gT@5[R1U<(RBd(DX#503?.[i?+)q>+qs:(1.?e=G=.m-E$bldfD]X)?Y[1^[C[>9`dc%1A.Z<G[k<+eL\1kWh:^8=O4;kq*5uQ<Y@a?DC9j9!HmOjMk)FsGjVU&8]$>mP"mb;@u%u-!%ls>NR6GqrF..?&L]W'c7CG/fRZ'2[rqlrfF5pJgAH]?hC<Ke[b%RTfrl+po!7\Q#2]2$%W5a'0o')TQm`pJ`WQ%n/*U99uEX0/pK(U&lN_s>W(2lC
 O_D0PCe-64>d*@oK%G1!G>RU7On7n2kE!s7llkP~>
+endstream
+endobj
+29 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 28 0 R
+/Annots 30 0 R
+>>
+endobj
+30 0 obj
+[
+31 0 R
+32 0 R
+]
+endobj
+31 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 599.266 194.988 587.266 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 11 0 R
+/H /I
+>>
+endobj
+32 0 obj
+<< /Type /Annot
+/Subtype /Link
+/Rect [ 108.0 582.066 148.668 570.066 ]
+/C [ 0 0 0 ]
+/Border [ 0 0 0 ]
+/A 25 0 R
+/H /I
+>>
+endobj
+33 0 obj
+<< /Length 2310 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+GatU5gMYb8&:O:Skg;j?L3H4;EuMYbS&B"c+@,7e=A^T*0ZdnIV.JiU41q13Q]fc@E$.d6cHM\4B4dKK<Yr6GMm!rJCe&t>?S"phSGBlps1;V5q3C,FL,[<NCjkjN`LH>1rUl^CZY]JL[Cn8pVbHr"bq*p553hru[;]cZGkTbNICA)Phd"E4g%4dTb70NL-_A"Vc2O<:I\imRDQ1H-cI/r@Asp,%Z>trIT?AqW5<%oY-CUQjrj1DOrZYu0>`$ldGY761j?/!#=9=5mV3pF+gVN1Fg$X_CH@[r_'"qaOe0$V6a7bI'Mb[c0dciXHMG3KnSiq)+OXD&SNW-PFb7u3>LeL2Kb9-d6&:KKX5Kc2W]XR&KW<7bi1iL3d6nS`>%@59ed0<O)&>*3[5G\D@iMn-,4$d7mko");#9QBA!YAf=7H[*f)3>#NZ9Gqlju2H#,QJ@+QZqodmNGk_Q6\T@Vkc)[\aI4'`%gJQ!QoZ&M)[cjD'grpdMSdbOrTO)A'kL&p\n+k5ai!j3L)^X8;Q\ZJY>o-6L)6o_*6pU;gQ:NQR(3r+q_?A-Hp$:Xp^)IjY+1*1%K[l%Eil&^C=u%_PkNi#%D#<,1"/L`1c*iOG#A]]_sMolR4<J_O-KZ>bR70MB"^;L5V(mC=nP\\8BkRN4C?I%WVFk%chMNX^<pFW/1TSVPV'A&iQ[^@U2;g71.4pHV(c(Vm<rU]_X1^eU4+E1'1"]'WF!0CMqdAi!5B-L<0quPKGG$KQU`>D2T[$+\&H@b+Dh,7_Fs*D>*8K<6WkI6mD[ZM80V<,[0NGTN^:hPaZPc\!Ned:ddut>Rht6'eEjOj?2&F^Up`IqZ!j&4&I%qr@&m>&H!uqDk*__QB+ld*K_NW&/9.[%9Kk!;-se^$Qck&YEa70E`n(Q<J,lM8:T<n5qAs"T'Rc+B,Sn$Q[GeQ38#EZa6cA"BF-^!>"fb0qF+-i&sAKaf^6GOe!pcHXHkKC#052dR:]f
 9i'WVLIk`L_b;bP3(b#+/glr^Yc[api.@009.Z-HWnIEW@dffImouMP56rq7+hjQ"7=j6b8dh"_+)$qdmf`u(&9#omgO,sd^6H_nBo"Gn)M8K.7qrOIE[l85Ds3UX*GP[b']CbC.&!e?0Q/YN6EZENWBWr(rWYne4T2S4!Xq4be%4K4f1-<#;q9Vl^-b/O$53K\H.=/-*4&X"*5XR5eFe&iI#oHQ-#olt7/=PCRTRmRe_lhBTNVf%rR6Z4k?ZlBk]Kc_6SuE/?DKEgLm8/;up(fSZs(BJ6aOW2D?6=m(Dh-PTV2uPZ@"0ZGH_aNRd=k0r02B)4Nn[mgT"gcPcM"=Q)<3+Xr0ctN#`PLP1T+o%OXldbW%CRAkH#Q*1kF(=b3>9!+?j%R[HhUCf/F\i<hKBb*,^ee4)s-O?1jV@/$("t4cNcDn%"76EJek.lo)88h/1<)2,tj..gk$nDC!hb7#B%\2PT/V\6H0,p:Xf$c:RnWFR6iNo)5RH&RZa@30[%GX2&G\\L-lfLhe6UU*15DrR9P`LdT*>W2#"4=q(QVoBtB<fsH.?!]@Q04^iQ1mUJFqU37<cJ81+Z<$8Sgl:KYT"Uh9L/loInF-]MA9q?WuGSo4-S&CJGg0rj8K(cQNedNlj7Uf[FdKU.$V!V\n%iOanpI-uN<K9g<:!k2\aW\k4!2tM^9;#He(/Ch(&MB9WFW)#33)$9BU"+rL;hmo(,QXgk1=IEJBYM97_a>%/b]3S:`+?00A5e8;'8V^CYa-uYjl&/fj`h0<W#^,sqL%kfrkUYH6X7qkQaZ`%Sd\FU+W;cs;uBFKGWjU'8%PWC(:-Ln$;/Y3alTg:^pOiB24sUuWLPlIX\1*AH,08cR4L;Ig1Ya]!pkjZ#Ps!"BM?QF@KB,Q[2G9WQ*T,2HAEfDI1Q#Ilas@OU%XkmnmMs/'/i\ueTm\9Kmeb]pV@F(fVRi3PI7_/<,RZ5iW%QZT]!X_3mQ(k*\\o<:m%
 TLF?u=nO$i#3R9U,2cI1lu=CE]DoZrIr=bDK^ZR_`SUKWB>I9_7AWjBZ2?)5UpXAFIq9TM`"*-8??1=NCd:hPSW#JD1AlAu1u$cPTKL%pr%_\?qE1:1)Ri@=.O;;&:0iF5Uu[WBVH/:4?VDCP6mmb=Q^&H@o.*3GO,?DEf>7-ip$F(=Vqmnc`rY25ghgGc'sf'(O>(@u*abh'KPC<\csZ";L30TCcLG<L<[ld=#3pLXf"gpcRFIM1m<7b(jC\phcn5gsCrAXhhV^M6YI?ehF[,$]U5>BsnZmeWD7ilJ"F4p)k,HYIn*g!8QF=+-q"E>*<k!]u-He,~>
+endstream
+endobj
+34 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 33 0 R
+>>
+endobj
+35 0 obj
+<< /Length 2398 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+Gatm=D/\/e&H88.Tl3SSQIWV/IMpQkft%o6[q-trAJf*;[^jPT,>*-m?b_eAR<K;.,X>5&0P0e9SpP_Vh"f4N`Qd5X_aWK8i:$Q$WR#rF9iLgO::eb)P:1!N&p^nSAf&3e]QnTcjSLJsYJ7YC]HDlGmcG>bI[1NE!q.g!8Qk?!hVH?Ol`q/X/+7&Z?fnX.]F_Igk@u<M)9SsVB!bTCr2sk[kH-:.1Ks@/_!ds3e[@48Yu^\K%'+P/'/_QV4pfi0gjIUl[4^uNj[.M'=g"p?^V.6$#=ucFMgH_RD-UkeW2prFQ$d8m#60+6Kc-"A&6?2B/'Z8hkforC"2D<SLQTnuYn@ekN\'*PKpjl(;H#,36p;!*+O41N=6Q0+@up:`ZJu_(DUT7,qsD6K@qcY**T->1f1WYDkKFmUiJgg%McZY$D(RFa*-"eT(dM)l6+b[7j`Z0+$1n&S0'#J1H2@Y'&#8"l]kZ_Bs(KgNRTGI[g1?W'LjD#"r+VMd%$+t.gm3oS[KdtD-7nIKpf"/aKNM_(+]rElOOXD&O@i/?Y5B8B[`.cIah\Lk:PX`rLC!gZ42iIT5t76j/MbG&XDo/">UKt_cs-+E_K\^:30k;q+cV.mpoK^+.9s>==*6*8j@krg#R<84U>G-(+I_d,h8k;cpssC6`"*gp@(3]bYks5F4CQ\[M\Z7cI+V@kZc@LCh<NiUf:=RJnI;55A$-+^'-+`@CNuG/2+HhL(?-`"Zrf<WcBYC;=b5bU$^q<S-tRU&R4XrIe'Na]b>"/RC=Z8`]c*<&P;XmlV'A\MaJ_]SiltK[i_,+^Q;o(eZ=n5O3Dfd1\XEUhd@!Y(5,2%8fPA>2!mop@:36TV71@4<iNt[AcrP\*J<3#uHm%CWW?o9+i'gKVVXq)m>I8G:i!5"%;Gta%17+*$EUog4($s_*CB5\TN%j!6$TXQ@A0dXH>lW;9.TS?*8/eA0'k<h$*:IbVg?X'4gFO*dTb1B_1!l-
 ^'Hh3@Hb0u8"np<C\ogh\Xu[@##3O>3jkVqrQ)g4l.L3<VL$^O279S=tH8ILAljqX@I6"a=h!uIhDkh8T+HgLF,g`8R2W%m#FH9JM4)c+4?AiL%"l8-.2@"nAODf>fMiD]"e9ls??#m9H&uI#Rh%Ao]1?*TX^p2=--;R;0TT\TA'V@Z2#L0)N!6bSC.56,qRg,Y%^\noV+SD;"TCokjfQ/1p1L+Ps5MpkbkP`ZugRDQjS"Z="J,+_#Mg8OdgUP/Vj]$EHlEeUt`5lUQ2%^:%dBiVeR8P1.;3IeuNOR%ZY-_m/AGH$#f?\D#P\t&:#V-.S/2cCKOSO/)L6P#pod#9@HU``Y;9I<HUB`&QPZN/m<%Ld?`dZ(IBhF`H$%:q=E>Yo:Ub`QPh[$^pM!%ua1siEcJq-4j\LEsiOh"cc=;OJgkF$Rf5fX%)'Ls\]_S_V"Kqm.Udc7PC)0^%o-t,t8l$GC`T/_0WoFq;M1^Sm?O?;8?.B)&J(F>:l0ZP=]o+pU,WbDZ?!#K[Y`GsZs(;AMf(oRH^]m[D5E>$EtLUc6#\%nHuQKuQ`ir.t]gFU[ai=P#g?:u]sD&.e$-mu\33#%q<L0eTJPPG+D7u(eGaf7eC+nCE<251:,Z*KfSCei=n4!Q;"Ag*(Y4#t(^,a%nQIAA0t;7Mtb%U#X[9$JN=gg;doa3j8q2]2rn$Iq'@]UU5q5#\R6g5@UH\%=>bZCj-n-(:Y[-lS?+Ul'+u"d1o=7:s+$g*,F=*C-55W)F>]j$]7;/;]B6gkT>/B\hLh3(9.>WgW-GP$lAiQZB>F)PKrrM,#d<5lb$3*]5s8=nFh?@E'bKTUP"JOYOFW1`0Ca^f\AaIp)u2<B#^4`5IoO<:=R'33L4f.nAE*>&.!FAZZo<L4$q5Hj/Vrb;)_1<XDV_JlP<])[O8:2F3+_7\fYqaOlplo%G,7N)3mr<ici4c`K@Xe:1\]eYC]ejNP>!Ze1iQ-7X/@+<kgFB<m
 cHcB6(;:g>Ek5XLic.Itdp-U%tpg]Nma/!P*-9@CR^o(?0c<_3ju',-ajQ?I+!"o,aH@Hsr"\*+H@f[/03KPkftYkk0_GJI*n83)7u68R7j<Q#_u0-s.cUpA*)Er*ALkeau1ZsSSFT4P>W16L^fK),TiZ)jl>-)"+/n=XaF8-?9s%?An@g+L?6m9YaiZ0QhlO@u%FpEAUb,sj7p0e#K<Le[ihRm9+JKMtO]FoomX5c&MsXoaifI?eCIJ+r!h!Dj'XPN8\8I240#d\$8+58`hCFhjb'nfZLpPN0(nO8\1hN\NS=i#f7oca%L&-g3MH+A(3B9tW"]$]Z;,:9A^tKTA8KL[P(',_%TWI6TN_A'@%DY1GeVj58A!fcFYHiQ3\4JAT7\j0XH2%tji;)ND_-(T3mu!5&73^I"0@~>
+endstream
+endobj
+36 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 35 0 R
+>>
+endobj
+37 0 obj
+<< /Length 2151 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+GatU5?'!_u&:D6"iO:.IUZbp*h_*$I\2J'L`^,,577#,(78t$C;6k9Aoqa7pR[$F7S0H%jZpf"Z\2ucF]pR^4c#:>OGa#Q7s3o6:FkfP.a80]k*Bl]GJsd9B_5-.U`Q`<[0-6Yq#*BS:j3M(&'@5:FbZ?akEL*[*H';iPj?'Uc:?J%fT2/KCcfLablKRAtIpD>3-/6bF/..tL:A`9uj6`N#WbP%E"cM;KMO-E1`EqUc4BjuXmV`#dm#Z:sjJC,Fk1B%pInf&iGEC4^LM(/I6]nfUl7Gc7Z?BbDE:JIm(J@W,gFlY0^MZ]TQ&=[\p]V24')^AfV=[qU!9E('!;K^S5'TosQ8G>pa,YX=UQPWU`p<f[,q@YX+7Lu==d-l5[T/_8'jVQT?tbO^1D#So=Z:BVZa2];&9bu"$r1A:+KR"j32ll\16H/n&#[Pe31Vc]%.m'b>Td:/F0GWh'"76U"tN?Oj3!0S2"R7%)56g'\0[80G=O2pWHSK?Q0l'iWIZ5^d2$+-gMs!ps*b\U==b-OX93iYICP?8!MEuWo'-tVe;o/TXX^A=a!8G8>#j(3!TN.MVGeWGA.Untg^=NrA(DJ[Nh_mnpTi)22QIVs^S;\16eBVkG-N/["*b)UIaCGkp7jek/tBod;5@T8.Q8;D*UPYmXGc7q'Fc3XO)AB##tmU6WF]4NbRo?)U2Y;"6DYo-Z%:XkRU\#gE6tHP_O'*=>7+K*b=ZC7\gI(lA8R5\Z5F+_bH]dC#L4S`1;IC)$C*Bk@7rCJqGBnKT/%>'K/nG03Yi`JaD)Z=B!HE&P`sM!9k_AbD`Bm9!,gGPi%5oWh[e(s0&_^nW/R2rm3,i]<p>GeS9V/le68P.+r2">$o;qKX,<uHJDse$AcP%c3aJ.M$$GVDNMYT)9h6/Of<JS97%aaD0DJW7DMW,2/=Hp0iU[`^]a*E+T:`u66t%i9V5pPU)P"/DeZauNV*LNV2f"(Rmlp^Kk3hrnh=(;
 EJVruX'F`t=Y$g`O#+9aCfQ+oke(5=[cZJ"#&Y2U*L?;@T;_E6a[KXQHWS5cXKFu;D9:?a/+q37m/+-hm7QLQQ(F[6Z&DLHcN)ZcFNJU!O440l-Ym"&->.h/d\Y\<,/MGdC2ut,P-,HK,V&R5CS0(P@5C`'+RW=:<rcU1_iUGOpUW\=h2o0R5`u.1d-c!$9$E$a?qsp'c@G;6=E/LO"SXjlZipkC62/D5\N.P3n,ZeCgbb0Bt![Ck/%i3)Q'#b-nl3LWe_lH[QN3a#E/apiePIK#6-'h$\4cZKgUYn1,@urWZX#l%0jh&!?n)s^$$MTrWk7?f`\>D!sLNd3,;X+PGU`r$`CCjg?cTr+j;a*-/!D+2JT'3kDkj^aD`nYNAZ--<dnN4;G)k&.U.bue(*,7&aU[+SmMk4hp@FA\j2!*K7ZAq4_ejUm$$@H+R)ao8YCEK`q)Dpnh)9]6+K4M`(fhj`d*e<prgk\:P)-Fi#;ANNg.XqGtZH!_Q(>mg62i-dXS3pC.nZBCGO$J5bGhRJX.!OW^@#RZY6kRD2iA&]Kh+eS=-"G[]0+QS8#.F[Gdt>*VWs#5\P&H^93dk#sb.H,6MN*XHb-Lp`]*Xf&"7r>E_K5@WS<-FO%-hncYN%k:r5NI.#lr$#K.$g9hK#IF\lH'7f.=Z%3^K.'?aM)PrbHu/6>Z.gYqq<N`eHDm:Mqi6&9EHq^=FVoj(nsUPDdtbf,N>1F:f6U<U6`_RMr)el]=lJYfN64/DYiC"AX!WJOBXL"rC[_TNmZJ6^d=VJl@YN\:F'0hL!ru,nBMU0@tH68!Df6X9"11FeUZ)_p!Xq_\TeZ%psD%Y:E$QJ:;EcD)>?*Bs]ef=F0EFB1;9td/Zt035HK(_*]L00:-p-d^X2,+RNEdN%Xk(rY^:ghZORdgh8IZ2-9SDC.I4JoT$AAd5o^N+epfDgL#c4,t3;=Fi#1A-(68GJ(?1DLQoPm^25Dn&ndNK^5dh4=7b
 FM<E_";]KC(JE>+na-Q7(I5<?%,lHFjOERS(S9U!qsmobi>QgH!$fQ.u=Fe,M(KFtl$p7fWr`R/4rOml.;]Th0+;QR2L]ZEJ*B]U%Um7;<lZlP^o[5<Nj/PC*]6*@6dl%Yl,k4R]EiJd#4=q_Fki4-7J0gGA9QO@42gj^%)QaQ~>
+endstream
+endobj
+38 0 obj
+<< /Type /Page
+/Parent 1 0 R
+/MediaBox [ 0 0 612 792 ]
+/Resources 3 0 R
+/Contents 37 0 R
+>>
+endobj
+39 0 obj
+<< /Length 1406 /Filter [ /ASCII85Decode /FlateDecode ]
+ >>
+stream
+GatU3gMZ%0&:O:Sk[>QV!J_8$?2IMFgU]ol_7#/N!kg_C&oA5G7#."9q!XObAS\662_Bsk-;AOCbaCKP4Dkp!D\fMam-OE&08nIOXqLqhrpT[GmrpsPOQV@YABoAgqr7/."u!h!q]=C/^\/[AoP^5T_JT"WqL]fpf-_tc4_I\QIsq5nLbF]-R/H%#Olj07"]L<HH+EC@KO-noVRdYa$_C;$WtO64g3l\9WYA7E\tcbsA5k:f7MH$0cmRuuBCi/)CQP)OKZ(s`i#:31lljsgS>99B58DiH8Jo)U^/Z-tcSt>7o%^ZPQTUrV(F>HoBN_V\J>3a%[shM1>lrpa-[T@o<+T+/3&i9#,rg\_RmNuGU2KmL?jY_)fCJhB@[42V-oSjh5bnRn'6u3j,@jqUM'u@M.5._NP[^\\Ht2COXhjll^1NID4GZh<1<0>@`4rJL#OAW62&g#c[bbmW?nu@OKmBRuR0**m;B1@aO;BFf+L<C%9S0[[Z6E+2bDI'ORYtf%RF'/1iGM2%Z_9l_Mq.Q!A20i+>\MG#bDg#OPH5]qp0'6UM9G*%3,-Q<$!/L5CE<MJ8!_L?f%Z>'0_J=lX=hMSH<N^?Uiphe7S=+HX_*F<%#]]:ZY;*o82l]tprIkbdcq8X%9X3iekRMp-2X.Q-CgT9?N13ZhQ"[0/iVZ0lCgj@I]f(n13F-\K!GOi4FI.Fd[@TI^WU^#DSO>R'9dNRPJH4L'ZFnZ$sWj&S(?M&-lkZX2.7VIVc;]Q!18!4TsdmEQ5.+Q0[rgE-pGEr#D^r#I[%LFhulGLMXp10jSB)%-X]E78?#3PmD9KFald]c7\e'BPUanE/_^Rl%<,Ljr[5Re';lW)9o1"l4!in!_>BgNc-CR:'[05><;QOcq>"arhekNC.n[V/%K$AHH``"'6#DJ7m.lC:Yu#e<FS+XG%e9lhG^,X\\O<cgn<OV;qb\?3?C,9TK*13*G4snFU*Z?soP'&2^*ra"l0f4
 lgW>@.o%oYEHG`/R"qdWDU-;q,Zf)/%Na.l!bB5AsBOZqUmK6HOAsNub!]7b'Nh^C;?]KhO#LpEN2lc"LD;u<kGVp+8*"Wh\OZAXK%jl7$'NCZ3dpdH5\fjFL]Xm`K3(.;O6*i.h(p2fE=F@A,/P+&VeD4LcJ7F(>+Os@u]h+l=$i07P!"N[-Q8&$]LI'$ji++5^,Q)G%3j?`[TN=aAUYB/n+"p`i=A*b7WPTo%l!M=>`F2%(j]kh[QLt#sD<Nu=_C!9?W**u*ER43PM:l0j_0rr0[^V,p:@+Z&CsDt1B'@cS!;sj>+kB'o$2#%\p"rm/)6lrbaO:E6>CPRNP'ilZ&Id!RX@q3dh2&RYe;`1_`XEmu1kF['Z)`n%q3m_ZW6;P^GS/S?'[[=-gBrn-kla::rXC8bq`F~>
+endstream
+endobj
+40 0 obj
+<</Type /XObject
+/Subtype /Image
+/Name /Im1
+/Length 15174
+/Width 494
+/Height 252
+/BitsPerComponent 8
+/ColorSpace /DeviceRGB
+/Filter /DCTDecode 
+>>
+stream
+ÿØÿà
Mime
View raw message