hadoop-zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r987314 [7/16] - in /hadoop/zookeeper/trunk: ./ src/contrib/hedwig/ src/contrib/hedwig/client/ src/contrib/hedwig/client/src/ src/contrib/hedwig/client/src/main/ src/contrib/hedwig/client/src/main/cpp/ src/contrib/hedwig/client/src/main/cpp...
Date Thu, 19 Aug 2010 21:25:22 GMT
Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/client/ssl/SslClientContextFactory.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/client/ssl/SslClientContextFactory.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/client/ssl/SslClientContextFactory.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/client/ssl/SslClientContextFactory.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.client.ssl;
+
+import javax.net.ssl.SSLContext;
+
+import org.apache.hedwig.client.conf.ClientConfiguration;
+
+public class SslClientContextFactory extends SslContextFactory {
+
+    public SslClientContextFactory(ClientConfiguration cfg) {
+        try {
+            // Create the SSL context.
+            ctx = SSLContext.getInstance("TLS");
+            ctx.init(null, getTrustManagers(), null);
+        } catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    @Override
+    protected boolean isClient() {
+        return true;
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/client/ssl/SslContextFactory.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/client/ssl/SslContextFactory.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/client/ssl/SslContextFactory.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/client/ssl/SslContextFactory.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.client.ssl;
+
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+public abstract class SslContextFactory {
+
+    protected SSLContext ctx;
+
+    public SSLContext getContext() {
+        return ctx;
+    }
+
+    protected abstract boolean isClient();
+
+    public SSLEngine getEngine() {
+        SSLEngine engine = ctx.createSSLEngine();
+        engine.setUseClientMode(isClient());
+        return engine;
+    }
+
+    protected TrustManager[] getTrustManagers() {
+        return new TrustManager[] { new X509TrustManager() {
+            // Always trust, even if invalid.
+
+            @Override
+            public X509Certificate[] getAcceptedIssuers() {
+                return new X509Certificate[0];
+            }
+
+            @Override
+            public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                // Always trust.
+            }
+
+            @Override
+            public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                // Always trust.
+            }
+        } };
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/conf/AbstractConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/conf/AbstractConfiguration.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/conf/AbstractConfiguration.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/conf/AbstractConfiguration.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.conf;
+
+import java.net.URL;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+
+public abstract class AbstractConfiguration {
+    protected CompositeConfiguration conf;
+
+    protected AbstractConfiguration() {
+        conf = new CompositeConfiguration();
+    }
+
+    /**
+     * You can load configurations in precedence order. The first one takes
+     * precedence over any loaded later.
+     * 
+     * @param confURL
+     */
+    public void loadConf(URL confURL) throws ConfigurationException {
+        Configuration loadedConf = new PropertiesConfiguration(confURL);
+        conf.addConfiguration(loadedConf);
+
+    }
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Callback.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Callback.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Callback.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Callback.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import org.apache.hedwig.exceptions.PubSubException;
+
+/**
+ * This class is used for callbacks for asynchronous operations
+ * 
+ */
+public interface Callback<T> {
+
+    /**
+     * This method is called when the asynchronous operation finishes
+     * 
+     * @param ctx
+     * @param resultOfOperation
+     */
+    public abstract void operationFinished(Object ctx, T resultOfOperation);
+
+    /**
+     * This method is called when the operation failed due to some reason. The
+     * reason for failure is passed in.
+     * 
+     * @param ctx
+     *            The context for the callback
+     * @param exception
+     *            The reason for the failure of the scan
+     */
+    public abstract void operationFailed(Object ctx, PubSubException exception);
+
+}
\ No newline at end of file

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/CallbackUtils.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/CallbackUtils.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/CallbackUtils.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/CallbackUtils.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import org.apache.hedwig.exceptions.PubSubException;
+import org.apache.hedwig.exceptions.PubSubException.CompositeException;
+
+public class CallbackUtils {
+
+    /**
+     * A callback that waits for all of a number of events to fire. If any fail,
+     * then fail the final callback with a composite exception.
+     * 
+     * TODO: change this to use any Exception and make CompositeException
+     * generic, not a PubSubException.
+     * 
+     * @param expected
+     *            Number of expected callbacks.
+     * @param cb
+     *            The final callback to call.
+     * @param ctx
+     * @param logger
+     *            May be null.
+     * @param level
+     *            Required iff logger != null.
+     * @param successMsg
+     *            If not null, then this is logged on success.
+     * @param failureMsg
+     *            If not null, then this is logged on failure.
+     * @param eagerErrorHandler
+     *            If not null, then this will be executed after the first
+     *            failure (but before the final failure callback). Useful for
+     *            releasing resources, etc. as soon as we know the composite
+     *            operation is doomed.
+     * @return
+     */
+    public static Callback<Void> multiCallback(final int expected, final Callback<Void> cb, final Object ctx,
+            final Logger logger, final Level level, final Object successMsg, final Object failureMsg,
+            Runnable eagerErrorHandler) {
+        if (expected == 0) {
+            cb.operationFinished(ctx, null);
+            return null;
+        } else {
+            return new Callback<Void>() {
+
+                final AtomicInteger done = new AtomicInteger();
+                final LinkedBlockingQueue<PubSubException> exceptions = new LinkedBlockingQueue<PubSubException>();
+
+                private void tick() {
+                    if (done.incrementAndGet() == expected) {
+                        if (exceptions.isEmpty()) {
+                            cb.operationFinished(ctx, null);
+                        } else {
+                            cb.operationFailed(ctx, new CompositeException(exceptions));
+                        }
+                    }
+                }
+
+                @Override
+                public void operationFailed(Object ctx, PubSubException exception) {
+                    if (logger != null && failureMsg != null)
+                        logger.log(level, failureMsg, exception);
+                    exceptions.add(exception);
+                    tick();
+                }
+
+                @Override
+                public void operationFinished(Object ctx, Void resultOfOperation) {
+                    if (logger != null && successMsg != null)
+                        logger.log(level, successMsg);
+                    tick();
+                }
+
+            };
+        }
+    }
+
+    /**
+     * A callback that waits for all of a number of events to fire. If any fail,
+     * then fail the final callback with a composite exception.
+     */
+    public static Callback<Void> multiCallback(int expected, Callback<Void> cb, Object ctx) {
+        return multiCallback(expected, cb, ctx, null, null, null, null, null);
+    }
+
+    /**
+     * A callback that waits for all of a number of events to fire. If any fail,
+     * then fail the final callback with a composite exception.
+     */
+    public static Callback<Void> multiCallback(int expected, Callback<Void> cb, Object ctx, Runnable eagerErrorHandler) {
+        return multiCallback(expected, cb, ctx, null, null, null, null, eagerErrorHandler);
+    }
+
+    private static Callback<Void> nop = new Callback<Void>() {
+
+        @Override
+        public void operationFailed(Object ctx, PubSubException exception) {
+        }
+
+        @Override
+        public void operationFinished(Object ctx, Void resultOfOperation) {
+        }
+
+    };
+
+    /**
+     * A do-nothing callback.
+     */
+    public static Callback<Void> nop() {
+        return nop;
+    }
+
+    /**
+     * Logs what happened before continuing the callback chain.
+     */
+    public static <T> Callback<T> logger(final Logger logger, final Level successLevel, final Level failureLevel, final Object successMsg,
+            final Object failureMsg, final Callback<T> cont) {
+        return new Callback<T>() {
+
+            @Override
+            public void operationFailed(Object ctx, PubSubException exception) {
+                logger.log(failureLevel, failureMsg, exception);
+                if (cont != null)
+                    cont.operationFailed(ctx, exception);
+            }
+
+            @Override
+            public void operationFinished(Object ctx, T resultOfOperation) {
+                logger.log(successLevel, successMsg);
+                if (cont != null)
+                    cont.operationFinished(ctx, resultOfOperation);
+            }
+
+        };
+    }
+
+    /**
+     * Logs what happened (no continuation).
+     */
+    public static Callback<Void> logger(Logger logger, Level successLevel, Level failureLevel, Object successMsg, Object failureMsg) {
+        return logger(logger, successLevel, failureLevel, successMsg, failureMsg, nop());
+    }
+
+    /**
+     * Return a Callback<Void> that just calls the given Callback cb with the
+     * bound result.
+     */
+    public static <T> Callback<Void> curry(final Callback<T> cb, final T result) {
+        return new Callback<Void>() {
+
+            @Override
+            public void operationFailed(Object ctx, PubSubException exception) {
+                cb.operationFailed(ctx, exception);
+            }
+
+            @Override
+            public void operationFinished(Object ctx, Void resultOfOperation) {
+                cb.operationFinished(ctx, result);
+            }
+
+        };
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/ConcurrencyUtils.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/ConcurrencyUtils.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/ConcurrencyUtils.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/ConcurrencyUtils.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CyclicBarrier;
+
+public class ConcurrencyUtils {
+
+    public static <T, U extends T, V extends BlockingQueue<T>> void put(V queue, U value) {
+        try {
+            queue.put(value);
+        } catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public static <T> T take(BlockingQueue<T> queue) {
+        try {
+            return queue.take();
+        } catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public static void await(CyclicBarrier barrier) {
+        try {
+            barrier.await();
+        } catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Either.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Either.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Either.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Either.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+public class Either<T, U> {
+
+    private T x;
+    private U y;
+
+    private Either(T x, U y) {
+        this.x = x;
+        this.y = y;
+    }
+
+    public static <T, U> Either<T, U> of(T x, U y) {
+        return new Either<T, U>(x, y);
+    }
+
+    public static <T, U> Either<T, U> left(T x) {
+        return new Either<T, U>(x, null);
+    }
+
+    public static <T, U> Either<T, U> right(U y) {
+        return new Either<T, U>(null, y);
+    }
+
+    public T left() {
+        return x;
+    }
+
+    public U right() {
+        return y;
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/FileUtils.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/FileUtils.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/FileUtils.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/FileUtils.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+public class FileUtils {
+
+    static DirDeleterThred dirDeleterThread;
+    static Logger log = Logger.getLogger(FileUtils.class);
+
+    static {
+        dirDeleterThread = new DirDeleterThred();
+        Runtime.getRuntime().addShutdownHook(dirDeleterThread);
+    }
+
+    public static File createTempDirectory(String prefix) throws IOException {
+        return createTempDirectory(prefix, null);
+    }
+
+    public static File createTempDirectory(String prefix, String suffix) throws IOException {
+        File tempDir = File.createTempFile(prefix, suffix);
+        if (!tempDir.delete()) {
+            throw new IOException("Could not delete temp file: " + tempDir.getAbsolutePath());
+        }
+
+        if (!tempDir.mkdir()) {
+            throw new IOException("Could not create temp directory: " + tempDir.getAbsolutePath());
+        }
+
+        dirDeleterThread.addDirToDelete(tempDir);
+        return tempDir;
+
+    }
+
+    static class DirDeleterThred extends Thread {
+        List<File> dirsToDelete = new LinkedList<File>();
+
+        public synchronized void addDirToDelete(File dir) {
+            dirsToDelete.add(dir);
+        }
+
+        @Override
+        public void run() {
+            synchronized (this) {
+                for (File dir : dirsToDelete) {
+                    deleteDirectory(dir);
+                }
+            }
+        }
+
+        protected void deleteDirectory(File dir) {
+            if (dir.isFile()) {
+                if (!dir.delete()) {
+                    log.error("Could not delete " + dir.getAbsolutePath());
+                }
+                return;
+            }
+
+            File[] files = dir.listFiles();
+            if (files == null) {
+                return;
+            }
+
+            for (File f : files) {
+                deleteDirectory(f);
+            }
+
+            if (!dir.delete()) {
+                log.error("Could not delete directory: " + dir.getAbsolutePath());
+            }
+
+        }
+
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/HedwigSocketAddress.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/HedwigSocketAddress.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/HedwigSocketAddress.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/HedwigSocketAddress.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import java.net.InetSocketAddress;
+
+/**
+ * This is a data wrapper class that is basically an InetSocketAddress with one
+ * extra piece of information for the SSL port (optional). This is used by
+ * Hedwig so we can encapsulate both regular and SSL port information in one
+ * data structure. Hedwig hub servers can be configured to listen on the
+ * standard regular port and additionally on an optional SSL port. The String
+ * representation of a HedwigSocketAddress is: <hostname>:<port>:<SSL
+ * port(optional)>
+ */
+public class HedwigSocketAddress {
+
+    // Member fields that make up this class.
+    private final String hostname;
+    private final int port;
+    private final int sslPort;
+
+    private final InetSocketAddress socketAddress;
+    private final InetSocketAddress sslSocketAddress;
+
+    // Constants used by this class.
+    public static final String COLON = ":";
+    private static final int NO_SSL_PORT = -1;
+
+    // Constructor that takes in both a regular and SSL port.
+    public HedwigSocketAddress(String hostname, int port, int sslPort) {
+        this.hostname = hostname;
+        this.port = port;
+        this.sslPort = sslPort;
+        socketAddress = new InetSocketAddress(hostname, port);
+        if (sslPort != NO_SSL_PORT)
+            sslSocketAddress = new InetSocketAddress(hostname, sslPort);
+        else
+            sslSocketAddress = null;
+    }
+
+    // Constructor that only takes in a regular port.
+    public HedwigSocketAddress(String hostname, int port) {
+        this(hostname, port, NO_SSL_PORT);
+    }
+
+    // Constructor from a String "serialized" version of this class.
+    public HedwigSocketAddress(String addr) {
+        String[] parts = addr.split(COLON);
+        this.hostname = parts[0];
+        this.port = Integer.parseInt(parts[1]);
+        if (parts.length > 2)
+            this.sslPort = Integer.parseInt(parts[2]);
+        else
+            this.sslPort = NO_SSL_PORT;
+        socketAddress = new InetSocketAddress(hostname, port);
+        if (sslPort != NO_SSL_PORT)
+            sslSocketAddress = new InetSocketAddress(hostname, sslPort);
+        else
+            sslSocketAddress = null;
+    }
+
+    // Public getters
+    public String getHostname() {
+        return hostname;
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    public int getSSLPort() {
+        return sslPort;
+    }
+
+    // Method to return an InetSocketAddress for the regular port.
+    public InetSocketAddress getSocketAddress() {
+        return socketAddress;
+    }
+
+    // Method to return an InetSocketAddress for the SSL port.
+    // Note that if no SSL port (or an invalid value) was passed
+    // during object creation, this call will throw an IllegalArgumentException
+    // (runtime exception).
+    public InetSocketAddress getSSLSocketAddress() {
+        return sslSocketAddress;
+    }
+
+    // Method to determine if this object instance is SSL enabled or not
+    // (contains a valid SSL port).
+    public boolean isSSLEnabled() {
+        return sslPort != NO_SSL_PORT;
+    }
+
+    // Return the String "serialized" version of this object.
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(hostname).append(COLON).append(port).append(COLON).append(sslPort);
+        return sb.toString();
+    }
+
+    // Implement an equals method comparing two HedwigSocketAddress objects.
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof HedwigSocketAddress))
+            return false;
+        HedwigSocketAddress that = (HedwigSocketAddress) obj;
+        return (this.hostname.equals(that.hostname) && (this.port == that.port) && (this.sslPort == that.sslPort));
+    }
+
+    // Static helper method to return the string representation for an
+    // InetSocketAddress. The HedwigClient can only operate in SSL or non-SSL
+    // mode. So the server hosts it connects to will just be an
+    // InetSocketAddress instead of a HedwigSocketAddress. This utility method
+    // can be used so we can store these server hosts as strings (ByteStrings)
+    // in various places (e.g. list of server hosts we've connected to
+    // or wrote to unsuccessfully).
+    public static String sockAddrStr(InetSocketAddress addr) {
+        return addr.getAddress().getHostAddress() + ":" + addr.getPort();
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Option.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Option.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Option.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Option.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+public class Option<T> {
+
+    private T x;
+
+    public static <T> Option<T> of(T x) {
+        return new Option<T>(x);
+    }
+
+    public static <T> Option<T> of() {
+        return new Option<T>();
+    }
+
+    public Option() {
+    }
+
+    public Option(T x) {
+        this.x = x;
+    }
+
+    public T get() {
+        return x;
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Pair.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Pair.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Pair.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/Pair.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+public class Pair<T, U> {
+
+    private T x;
+    private U y;
+
+    public Pair(T x, U y) {
+        this.x = x;
+        this.y = y;
+    }
+
+    public static <T, U> Pair<T, U> of(T x, U y) {
+        return new Pair<T, U>(x, y);
+    }
+
+    public T first() {
+        return x;
+    }
+
+    public U second() {
+        return y;
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/PathUtils.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/PathUtils.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/PathUtils.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/java/org/apache/hedwig/util/PathUtils.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+public class PathUtils {
+
+    /** Generate all prefixes for a path. "/a/b/c" -> ["/a","/a/b","/a/b/c"] */
+    public static List<String> prefixes(String path) {
+        List<String> prefixes = new ArrayList<String>();
+        String prefix = "";
+        for (String comp : path.split("/+")) {
+            // Skip the first (empty) path component.
+            if (!comp.equals("")) {
+                prefix += "/" + comp;
+                prefixes.add(prefix);
+            }
+        }
+        return prefixes;
+    }
+
+    /** Return true iff prefix is a prefix of path. */
+    public static boolean isPrefix(String prefix, String path) {
+        String[] as = prefix.split("/+"), bs = path.split("/+");
+        if (as.length > bs.length)
+            return false;
+        for (int i = 0; i < as.length; i++)
+            if (!as[i].equals(bs[i]))
+                return false;
+        return true;
+    }
+
+    /** Like File.getParent but always uses the / separator. */
+    public static String parent(String path) {
+        return new File(path).getParent().replace("\\", "/");
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/resources/log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/resources/log4j.properties?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/resources/log4j.properties (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/main/resources/log4j.properties Thu Aug 19 21:25:13 2010
@@ -0,0 +1,32 @@
+#
+# 
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# 
+#
+log4j.rootLogger=INFO, A1
+log4j.logger.org.apache.zookeeper = ERROR
+log4j.logger.org.apache.bookkeeper.client.QuorumOpMonitor = ERROR
+log4j.logger.org.apache.bookkeeper.proto.BookieClient = ERROR
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%d %-4r [%t] %-5p %c %x - %m%n
+

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/client/AppTest.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/client/AppTest.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/client/AppTest.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/client/AppTest.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.client;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit test for simple App.
+ */
+public class AppTest extends TestCase {
+    /**
+     * Create the test case
+     * 
+     * @param testName
+     *            name of the test case
+     */
+    public AppTest(String testName) {
+        super(testName);
+    }
+
+    /**
+     * @return the suite of tests being tested
+     */
+    public static Test suite() {
+        return new TestSuite(AppTest.class);
+    }
+
+    /**
+     * Rigourous Test :-)
+     */
+    public void testApp() {
+        assertTrue(true);
+    }
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestFileUtils.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestFileUtils.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestFileUtils.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestFileUtils.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import java.io.File;
+
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+public class TestFileUtils extends TestCase {
+
+    @Test
+    public void testCreateTmpDirectory() throws Exception {
+        String prefix = "abc";
+        String suffix = "def";
+        File dir = FileUtils.createTempDirectory(prefix, suffix);
+        assertTrue(dir.isDirectory());
+        assertTrue(dir.getName().startsWith(prefix));
+        assertTrue(dir.getName().endsWith(suffix));
+        FileUtils.dirDeleterThread.start();
+        FileUtils.dirDeleterThread.join();
+        assertFalse(dir.exists());
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestHedwigSocketAddress.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestHedwigSocketAddress.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestHedwigSocketAddress.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestHedwigSocketAddress.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import java.net.InetSocketAddress;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+
+public class TestHedwigSocketAddress extends TestCase {
+
+    // Common values used by tests
+    private String hostname = "localhost";
+    private int port = 4080;
+    private int sslPort = 9876;
+    private int invalidPort = -9999;
+    private String COLON = ":";
+    
+    @Test
+    public void testCreateWithSSLPort() throws Exception {
+        HedwigSocketAddress addr = new HedwigSocketAddress(hostname, port, sslPort);
+        assertTrue(addr.getSocketAddress().equals(new InetSocketAddress(hostname, port)));
+        assertTrue(addr.getSSLSocketAddress().equals(new InetSocketAddress(hostname, sslPort)));
+    }
+
+    @Test
+    public void testCreateWithNoSSLPort() throws Exception {
+        HedwigSocketAddress addr = new HedwigSocketAddress(hostname, port);
+        assertTrue(addr.getSocketAddress().equals(new InetSocketAddress(hostname, port)));
+        assertTrue(addr.getSSLSocketAddress() == null);
+    }
+
+    @Test
+    public void testCreateFromStringWithSSLPort() throws Exception {
+        HedwigSocketAddress addr = new HedwigSocketAddress(hostname+COLON+port+COLON+sslPort);
+        assertTrue(addr.getSocketAddress().equals(new InetSocketAddress(hostname, port)));
+        assertTrue(addr.getSSLSocketAddress().equals(new InetSocketAddress(hostname, sslPort)));
+    }    
+
+    @Test
+    public void testCreateFromStringWithNoSSLPort() throws Exception {
+        HedwigSocketAddress addr = new HedwigSocketAddress(hostname+COLON+port);
+        assertTrue(addr.getSocketAddress().equals(new InetSocketAddress(hostname, port)));
+        assertTrue(addr.getSSLSocketAddress() == null);
+    }
+    
+    @Test
+    public void testCreateWithInvalidRegularPort() throws Exception {
+        boolean success = false;
+        try {
+            new HedwigSocketAddress(hostname+COLON+invalidPort);
+        }
+        catch (IllegalArgumentException e) {
+            success = true;
+        }
+        assertTrue(success);
+    }    
+
+    @Test
+    public void testCreateWithInvalidSSLPort() throws Exception {
+        boolean success = false;
+        try {
+            new HedwigSocketAddress(hostname, port, invalidPort);
+        }
+        catch (IllegalArgumentException e) {
+            success = true;
+        }
+        assertTrue(success);
+    }    
+
+    @Test
+    public void testToStringConversion() throws Exception {
+        HedwigSocketAddress addr = new HedwigSocketAddress(hostname, port, sslPort);
+        HedwigSocketAddress addr2 = new HedwigSocketAddress(addr.toString());
+        assertTrue(addr.getSocketAddress().equals(addr2.getSocketAddress()));
+        assertTrue(addr.getSSLSocketAddress().equals(addr2.getSSLSocketAddress()));
+        addr.toString().equals(addr2.toString());
+    }
+
+    @Test
+    public void testIsSSLEnabledFlag() throws Exception {
+        HedwigSocketAddress sslAddr = new HedwigSocketAddress(hostname, port, sslPort);
+        assertTrue(sslAddr.isSSLEnabled());
+        HedwigSocketAddress addr = new HedwigSocketAddress(hostname, port);
+        assertFalse(addr.isSSLEnabled());               
+    }
+    
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestPathUtils.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestPathUtils.java?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestPathUtils.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/client/src/test/java/org/apache/hedwig/util/TestPathUtils.java Thu Aug 19 21:25:13 2010
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hedwig.util;
+
+import java.util.Arrays;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+
+public class TestPathUtils extends TestCase {
+
+    @Test
+    public void testPrefixes() {
+        assertEquals(Arrays.asList(new String[] { "/a", "/a/b", "/a/b/c" }), PathUtils.prefixes("/a/b/c"));
+        assertEquals(Arrays.asList(new String[] { "/a", "/a/b", "/a/b/c" }), PathUtils.prefixes("///a///b///c"));
+
+    }
+
+    @Test
+    public void testIsPrefix() {
+        String[] paths = new String[] { "/", "/a", "/a/b" };
+        for (int i = 0; i < paths.length; i++) {
+            for (int j = 0; j <= i; j++) {
+                assertTrue(PathUtils.isPrefix(paths[j], paths[i]));
+                assertTrue(PathUtils.isPrefix(paths[j], paths[i] + "/"));
+                assertTrue(PathUtils.isPrefix(paths[j] + "/", paths[i]));
+                assertTrue(PathUtils.isPrefix(paths[j] + "/", paths[i] + "/"));
+            }
+            for (int j = i + 1; j < paths.length; j++) {
+                assertFalse(PathUtils.isPrefix(paths[j], paths[i]));
+                assertFalse(PathUtils.isPrefix(paths[j], paths[i] + "/"));
+                assertFalse(PathUtils.isPrefix(paths[j] + "/", paths[i]));
+                assertFalse(PathUtils.isPrefix(paths[j] + "/", paths[i] + "/"));
+            }
+        }
+    }
+
+}

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/conf/hw_client_sample.conf
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/conf/hw_client_sample.conf?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/conf/hw_client_sample.conf (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/conf/hw_client_sample.conf Thu Aug 19 21:25:13 2010
@@ -0,0 +1,7 @@
+# The default Hedwig server host to contact (this ideally should be a VIP 
+# that fronts all of the Hedwig server hubs).
+default_server_host=hwServer1:4080:9876
+# This parameter is a boolean flag indicating if communication with the
+# server should be done via SSL for encryption. The Hedwig server hubs also
+# need to be SSL enabled for this to work.
+ssl_enabled=false

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/conf/hw_server_sample.conf
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/conf/hw_server_sample.conf?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/conf/hw_server_sample.conf (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/conf/hw_server_sample.conf Thu Aug 19 21:25:13 2010
@@ -0,0 +1,10 @@
+# The ZooKeeper server host(s) for the Hedwig Server to use.
+zk_host=zkServer1:2181
+# The number of milliseconds of each tick in ZooKeeper.
+zk_timeout=2000
+# The port at which the clients will connect.
+server_port=4080
+# The SSL port at which the clients will connect (only if SSL is enabled).
+ssl_server_port=9876
+# Flag indicating if the server should also operate in SSL mode.
+ssl_enabled=false

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/doc/build.txt
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/doc/build.txt?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/doc/build.txt (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/doc/build.txt Thu Aug 19 21:25:13 2010
@@ -0,0 +1,146 @@
+% Building Hedwig
+% Yang Zhang
+
+Pre-requisites
+==============
+
+For the core itself:
+
+- JDK 6: <http://java.sun.com/>.  Ensure `$JAVA_HOME` is correctly set.
+- Maven 2: <http://maven.apache.org/>.
+- Protocol Buffers 2.3.0: <http://protobuf.googlecode.com/>.
+- Zookeeper 3.4.0: <http://hadoop.apache.org/zookeeper/>.  See below.
+- Bookkeeper 3.4.0: <http://hadoop.apache.org/zookeeper/>.  See below.
+
+Hedwig has been tested on Windows XP, Linux 2.6, and OS X.
+
+For the deployment and distributed support scripts in `hw.bash`:
+
+- Ant: <http://ant.apache.org/>, if you want to build Zookeeper.
+- Bash: <http://www.gnu.org/software/bash/>.
+- Coreutils: <http://www.gnu.org/software/coreutils/>.
+- Expect: <http://expect.nist.gov/>, if you want `unbuffer`.
+- Findutils: <http://www.gnu.org/software/findutils/>.
+- OpenSSH: <http://www.openssh.com/>.
+- Python 2.6: <http://python.org/>.
+
+Protocol Buffers
+----------------
+
+Hedwig requires the use of the Java runtime libraries of Protocol Buffers 2.3.0.
+These libraries need to be installed into your local maven repository. (Maven allows
+multiple versions to be installed.) To install protocol buffels to your local
+repository, you have to download the tarball and follow the README.txt 
+instructions. Note that you must first install the C++ package which contains the
+compiler (protoc) before you can build the java libraries. That will install the
+library jar's in the local maven repository where Hedwig is currently configured
+to point to.
+
+Zookeeper and Bookkeeper
+------------------------
+
+Hedwig currently requires the version of Bookkeeper maintained in Apache's current
+trunk SVN respository (version 3.4.0). This is not a released version yet but certain
+features needed for BookKeeper are only available there.
+
+Hedwig also depends on ZK testing code for its own testing code.
+
+Since Hedwig is a Maven project, all these dependencies must be made available
+as Maven artifacts.  However, neither ZK nor BK are currently Mavenized.
+Hedwig provides some bash scripts to ease the installation of ZK, ZK tests, and
+BK, all as Maven artifacts.
+
+Currently, we have included the necessary ZooKeeper and BookKeeper jars in the Hedwig
+source itself in the $HEDWIG_DIR/server/lib directory. There is no need to retrieve
+them directly from the Apache download site as they are non-released trunk versions.
+
+#Not relevant right now since we already have the ZK jars already in the Hedwig source.
+To fetch and build ZK 3.4.0 (and its tests) in the current directory, run:
+
+  $HEDWIG_DIR/scripts/hw.bash get-zk
+
+#Not relevant right now, but when we start using the apache version of BK, to
+build the local version of BK:
+
+  $HEDWIG_DIR/scripts/hw.bash get-bk
+
+The $HEDWIG_DIR/server/lib directory contains all of the the class and source jars for
+ZK, ZK tests, and BK.  To install these, go to that directory and run the following
+command to install them into your local maven repository:
+
+  $HEDWIG_DIR/scripts/hw.bash install-zk-bk
+
+Command-Line Instructions
+=========================
+
+From the main Hedwig directory, run `mvn package`.  This will produce the
+executable jars for both the client and server, as well as a server "assembly
+jar" containing all dependencies as well for easier deployment.
+
+See the User's Guide for instructions on running and usage.
+
+Eclipse Instructions
+====================
+
+To check out, build, and develop using Eclipse:
+
+1. Install the Subclipse plugin.  Update site:
+   <http://subclipse.tigris.org/update_1.4.x>.
+
+2. Install the Maven plugin.  Update site:
+   <http://m2eclipse.sonatype.org/update>.  From the list of packages available
+   from this site, select everything under the "Maven Integration" category,
+   and from the optional components select the ones with the word "SCM" in them.
+
+3. Go to Preferences > Team > SVN.  For the SVN interface, choose "Pure Java".
+
+4. Choose File > New > Project... > Maven > Checkout Maven Projects from SCM.
+
+5. For the SCM URL type, choose SVN.  For the URL, enter
+   SVN URL.  Maven will automatically
+   create a top-level Eclipse project for each of the 4 Maven modules
+   (recommended).  If you want fewer top-level projects, uncheck the option of
+   having a project for each module (under Advanced).
+
+6. Right-click on the `protocol` project and choose Run As > Maven
+   generate-sources.  This will generate the Java and C++ code for Protocol
+   Buffers.
+
+7. Refresh the workspace to pick up the generated code and add
+   `hedwig/protocol/target/generated-sources/java` as a source folder.  (6 & 7
+   should really be doable automatically, but I haven't figured out how.)
+
+You are now ready to run and debug the client and server code.  See the User's
+Guide for instructions on running and usage.
+
+Utilities
+=========
+
+Removing Conflicting Files in Jars
+----------------------------------
+
+The Maven assembly plugin that produces the fat assembly jar may end up putting
+into the jar files with the same conflicting paths from multiple dependencies.
+This makes working with the files from certain tools (like `jar`) a bit jarring.
+In our case, these files are not things like class files, but rather README and
+LICENSE files, so we can safely remove conflicts by choosing an arbitrary winner.
+To do so, run:
+
+  $HEDWIG_DIR/scripts/hw.bash strip-jar
+
+Adjusting Logging
+-----------------
+
+The logging level is something that is baked into the jar in the
+`log4j.properties` resource.  However, it would be wasteful to go through a
+Maven build cycle to update and adjust this.  If you're working from a source
+tree, it's also annoying to have to edit a source file to adjust the logging.
+
+We have a little script for tweaking the logging level.  After running
+`strip-jar`, run:
+
+  $HEDWIG_DIR/scripts/hw.bash set-logging WARN
+
+To see what the current logging level is:
+
+  $HEDWIG_DIR/scripts/hw.bash get-logging

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/doc/dev.txt
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/doc/dev.txt?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/doc/dev.txt (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/doc/dev.txt Thu Aug 19 21:25:13 2010
@@ -0,0 +1,338 @@
+% Developer's Guide
+
+Style
+=====
+
+We have provided an Eclipse Formatter file `formatter.xml` with all the
+formatting conventions currently used in the project.  Highlights include no
+tabs, 4-space indentation, and 120-char width.  Please respect this so as to
+reduce the amount of formatting-related noise produced in commits.
+
+Static Analysis
+===============
+
+We would like to use static analysis tools PMD and FindBugs to maintain code
+quality.  However, we have not yet arrived at a consensus on what rules to
+adhere to, and what to ignore.
+
+Netty Notes
+===========
+
+The asynchronous network IO infrastructure that Hedwig uses is [Netty].  Here
+are some notes on Netty's concurrency architecture and its filter pipeline
+design.
+
+[Netty]: http://www.jboss.org/netty
+
+Concurrency Architecture
+------------------------
+
+After calling `ServerBootstrap.bind()`, Netty starts a boss thread
+(`NioServerSocketPipelineSink.Boss`) that just accepts new connections and
+registers them with one of the workers from the `NioWorker` pool in round-robin
+fashion (pool size defaults to CPU count).  Each worker runs its own select
+loop over just the set of keys that have been registered with it.  Workers
+start lazily on demand and run only so long as there are interested fd's/keys.
+All selected events are handled in the same thread and sent up the pipeline
+attached to the channel (this association is established by the boss as soon as
+a new connection is accepted).
+
+All workers, and the boss, run via the executor thread pool; hence, the
+executor must support at least two simultaneous threads.
+
+Handler Pipeline
+----------------
+
+A pipeline implements the intercepting filter pattern.  A pipeline is a
+sequence of handlers.  Whenever a packet is read from the wire, it travels up
+the stream, stopping at each handler that can handle upstream events.
+Vice-versa for writes.  Between each filter, control flows back through the
+centralized pipeline, and a linked list of contexts keeps track of where we are
+in the pipeline (one context object per handler).
+
+Distributed Performance Evaluation
+==================================
+
+We've included some scripts to repeatedly run varying configurations of Hedwig
+on a distributed testbed and collect the resulting data.  The experiments use
+the `org.apache.hedwig.client.App` client application and are driven by
+`scripts/hw.bash` (via the `app` command).
+
+Currently, we have two types of experiments: subscription benchmarks and
+publishing benchmarks.
+
+Subscription Benchmarks
+-----------------------
+
+The subscription benchmark compares synchronous and asynchronous subscriptions.
+Because the synchronicity of subscriptions is a server configuration parameter,
+the servers must be restarted to change this.  The benchmarks varies the
+maximum number of outstanding subscription requests.
+
+To run the subscription benchmark with wilbur6 as the subscriber and wilbur1 as
+its default hub:
+
+  hosts=wilbur6 scripts/hw.bash sub-exp wilbur1
+
+This produces log files into the `sub` directory, which may then be analyzed
+using the analysis scripts.
+
+Publishing Benchmarks
+---------------------
+
+The publishing benchmark measures the throughput and latency of publishing
+messages within a LAN and across a WAN.  It varies the following parameters:
+
+- maximum number of outstanding publish requests
+- number of publishers
+- number of (local) receivers
+
+We vary each dimension separately (and have default settings) to avoid a
+combinatorial explosion in the number of configurations to explore.
+
+First, start a (standalone) instance:
+
+  scripts/hw.bash start-region '' $hwhost $zkhost $bk1host $bk2host $bk3host
+
+To run this over `$host1` through `$host3`, with the number of
+publishers/subscribers varying linearly over this set:
+
+  npars="20 40 60 80 100" scripts/hw.bash pub-exps "$host1 $host2 $host3" $hwhost $zkhost
+
+This will vary the number of outstanding publish requests as specified in
+`npars`.
+
+You may also optionally run this experiment with a second subscribing region:
+
+  scripts/hw.bash start-zk-bks $zkhost $bk1host $bk2host $bk3host
+  npars="..." scripts/hw.bash pub-exps "$host1 $host2 $host3" $hwhost $zkhost $rrecv $rhwhost $rzkhost
+
+where the final three extra arguments specify the client receiver, Hedwig, and
+Zookeeper hosts, in that order.
+
+This command will produce files into `./pub/`, which can then be process using
+`analyze.py`.
+
+Analysis and Visualization
+==========================
+
+`scripts/analyze.py` produces plots from the collected experimental data.  It
+has just a few immediate dependencies. In the following, the
+indentation signifies nested dependencies, like an upside-down tree:
+
+      component AAA that component AA requires
+      component AAB that component AA requires
+    component AA that component A requires
+      component ABA that component AB requires
+      component ABB that component AB requires
+    component AB that component A requires
+  component A that analysis tools depend on
+      component BAA that component BA requires
+      component BAB that component BA requires
+    component BA that component B requires
+      component BBA that component BB requires
+      component BBB that component BB requires
+    component BB that component B requires
+  component B that analysis tools depend on
+
+The reason the tree is upside-down is so that you can treat this whole thing as
+a chunk of bash script.
+
+[toast] is a utility that makes it a breeze to install all this software, but
+you do need to make sure your environment is set up correctly (e.g.
+`PKG_CONFIG_PATH` must point to `~/.toast/armed/lib/pkgconfig/`).
+
+Setup:
+
+  wget -O- http://toastball.net/toast/toast|perl -x - arm toast
+
+  toast arm "http://www.python.org/ftp/python/2.6.2/Python-2.6.2.tar.bz2"
+
+  toast arm numpy
+
+        toast arm libpng
+
+        toast arm pixman
+
+        toast arm freetype
+
+          toast arm 'ftp://xmlsoft.org/libxml2/libxml2-2.7.3.tar.gz'
+
+        toast arm fontconfig
+
+      toast arm cairo
+
+    toast arm pycairo
+
+  hg clone https://yang@bitbucket.org/yang/pycha/
+  pycha/setup.bash -d -p $path_to_install_to
+
+  svn co https://assorted.svn.sourceforge.net/svnroot/assorted/python-commons/trunk/ python-commons/
+  python-commons/setup.bash -d -p $path_to_install_to
+
+To analyze the publishing experiments, change to the `pub` data directory and
+run:
+
+  scripts/analyze.py pub
+
+To analyze the subscription experiments, change to the `sub` data directory
+and run:
+
+  scripts/analyze.py sub
+
+[toast]: http://toastball.net/toast/
+
+Debugging
+=========
+
+You can attach an Eclipse debugger (or any debugger) to a Java process running
+on a remote host, as long as it has been started with the appropriate JVM
+flags.  (See the Building Hedwig document to set up your Eclipse environment.)
+To launch something using `hw.bash` with debugger attachment enabled, prefix
+the command with `attach=true`, e.g.:
+
+  attach=true scripts/hw.bash start-regions myregions.cfg
+
+Profiling
+=========
+
+The scripts we have provided include ways for you to launch with YourKit
+profiling enabled.
+
+To deploy YourKit onto a number of machines:
+
+  hosts="..." scripts/hw.bash setup-yjp $path_to_yjp
+
+where the path points to the [YourKit Linux zip archive] (which is freely
+available and doesn't require any license to use).
+
+Now when using the scripts to run distributed experiments, to profile anything
+with YourKit, prefix the command with `use_yjp=true`.  E.g.:
+
+  use_yjp=true scripts/hw.bash start-regions regions.cfg
+
+Now you may start on your local machine the YourKit GUI and connect to the
+hosts that you're interested in.
+
+Note that you may want to disable the default set of filters in YourKit.
+
+[YourKit Linux zip archive]: http://www.yourkit.com/download/yjp-8.0.15.zip
+
+Pseudocode
+==========
+
+This summarizes the control flow through the system.
+
+  publishhandler
+    topicmgr.getowner
+      (maybe) claim the topic, calling back into persmgr.acquiredtopic
+        read /hedwig/standalone/topics/TOPIC (which should initially be empty)
+        for each line, parse as "STARTSEQ\tLEDGERID" # TODO how is this written?
+          ledger = bk.openledger(ledgerid)
+          lastid = ledger.getlast
+          if lastid > 0, lrs[startseq] = persmgr.ledger2lr[ledgerid] = new LedgerRange(ledger, ledgerid, startseq, startseq + lastid # TODO what are ledger ranges?
+        create new ledger for topic
+          # TODO read
+          lr = new LedgerRange(ledger, ledgerid, lastid, -1)
+          lrs[lastid] = lr
+          persmgr.topic2ranges[topic] = lrs
+    add region info to pub req and send that to persmgr.persistmessage
+      entryid = persmgr.topic2ranges[topic].last.ledger.addentry(the pub'd data)
+      update persmgr.topic2lastseq[topic]:
+        .local = persmgr.ledger2lr[ledger id].startseq + entryid
+        .regions = maxes of orig seq and incoming pub seq
+
+  subscribehandler
+    topicmgr.getowner...
+    delivmgr.startservingsubscription(topic, endpoint, ishubsubscriber)
+      delivmgr.endpoint2sub[endpoint] = new subscriber(lastseq = persmgr.getcurrentseqidfortopic(topic).local)
+      delivmgr.topic2ptr2subs[topic][ptr].add(sub)
+      sub.delivernextmessage
+        sub.curseq = persmgr.getseqidafterskipping(topic, sub.lastseq, skip = 1)
+        msg = persmgr.scansinglemessage(topic, seq = sub.curseq)
+          if persmgr.topic2lastseq[topic].local >= seq
+            lr = persmgr.topic2ranges[topic].floor(seq)
+            return lr.ledger.read(first = last = seq - lr.startseq)
+        if failed, then retry in 1 s
+        endpoint.send(msg)
+        movedeliveryptr
+          delivmgr.topic2ptr2subs[topic][sub.lastseq].remove(sub)
+          delivmgr.topic2ptr2subs[topic][sub.curseq].add(sub)
+        previd = sub.lastseq, sub.lastseq = sub.curseq
+        sub.delivernextmessage...
+
+ReadAhead Cache
+================
+
+The delivery manager class is responsible for pushing published messages from 
+the hubs to the subscribers. The most common case is that all subscribers are 
+connected and either caught up, or close to the tail end of the topic. In this 
+case, we don't want the delivery manager to be polling bookkeeper for any newly 
+arrived messages on the topic; new messages should just be pushed to the 
+delivery manager. However, there is also the uncommon case when a subscriber is 
+behind, and messages must be pulled from Bookkeeper.
+
+Since all publishes go through the hub, it is possible to cache the recently 
+published messages in the hub, and then the delivery manager won't have to make 
+the trip to bookkeeper to get the messages but instead get them from local 
+process memory.
+
+These ideas of push, pull, and caching are unified in the following way:
+- A hub has a cache of messages
+
+- When the delivery manager wants to deliver a message, it asks the cache for 
+  it. There are 3 cases:
+  - The message is available in the cache, in which case it is given to the 
+    delivery manager
+  - The message is not present in the cache and the seq-id of the message is 
+    beyond the last message published on that topic (this happens if the 
+    subscriber is totally caught up for that topic). In this case, a stub is put 
+    in the cache in order to notify the delivery manager when that message does 
+    happen to be published.
+  - The message is not in the cache but has been published to the topic. In this 
+    case, a stub is put in the cache, and a read is issued to bookkeeper.
+
+- Whenever a message is published, it is cached. If there is a stub already in 
+  the cache for that message, the delivery manager is notified. 
+
+- Whenever a message is read from bookkeeper, it is cached. There must be a stub 
+  for that message (since reads to bookkeeper are issued only after putting a 
+  stub), so the delivery manager is notified. 
+
+- The cache does readahead, i.e., if a message requested by the delivery manager 
+  is not in the cache, a stub is established not only for that message, but also 
+  for the next n messages where n is configurable (default 10). On a cache hit, 
+  we look ahead n/2 messages, and if that message is not present, we establish 
+  another n/2 stubs. In short, we always ensure that the next n stubs are always 
+  established.
+
+- Over time, the cache will grow in size. There are 2 pruning mechanisms:
+  
+  - Once all subscribers have consumed up to a particular seq-id, they notify 
+    the cache, and all messages up to that seq-id are pruned from the cache.
+  - If the above pruning is not working (e.g., because some subscribers are 
+    down), the cache will eventually hit its size limit which is configurable  
+    (default, half of maximum jvm heap size). At this point, messages are just 
+    pruned in FIFO order. We use the size of the blobs in the message for 
+    estimating the cache size. The assumption is that that size will dominate 
+    over fixed, object-level size overheads. 
+  - Stubs are not purged because according to the above simplification, they are 
+    of 0 size.
+
+Scalability Bottlenecks Down the Road
+=====================================
+
+- Currently each topic subscription is served on a different channel. The number 
+  of channels will become a bottleneck at higher channels. We should switch to 
+  an architecture, where multiple topic subscriptions between the same client, 
+  hub pair should be served on the same channel. We can have commands to start, 
+  stop subscriptions sent all the way to the server (right now these are local).
+- Publishes for a topic are serialized through a hub, to get ordering 
+  guarantees. Currently, all subscriptions to that topic are served from the 
+  same hub. If we start having large number of subscribers to heavy-volume 
+  topics, the outbound bandwidth at the hub, or the CPU at that hub might become 
+  the bottleneck. In that case, we can setup other regions through which the 
+  messages are routed (this hierarchical scheme) reduces bandwidth requirements 
+  at any single node. It should be possible to do this entirely through 
+  configuration.
+

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/doc/doc.txt
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/doc/doc.txt?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/doc/doc.txt (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/doc/doc.txt Thu Aug 19 21:25:13 2010
@@ -0,0 +1,17 @@
+% Meta-Documentation
+
+In the documentation directory, you'll find:
+
+- `build.txt`: Building Hedwig, or how to set up Hedwig
+- `user.txt`: User's Guide, or how to program against the Hedwig API and how to
+  run it
+- `dev.txt`: Developer's Guide, or Hedwig internals and hacking details
+
+These documents are all written in the [Pandoc] dialect of [Markdown].  This
+makes them readable as plain text files, but also capable of generating HTML or
+LaTeX documentation.
+
+[Pandoc]: http://johnmacfarlane.net/pandoc/
+[Markdown]: http://daringfireball.net/projects/markdown/
+
+Documents are wrapped at 80 chars and use 2-space indentation.

Added: hadoop/zookeeper/trunk/src/contrib/hedwig/doc/user.txt
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/hedwig/doc/user.txt?rev=987314&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/hedwig/doc/user.txt (added)
+++ hadoop/zookeeper/trunk/src/contrib/hedwig/doc/user.txt Thu Aug 19 21:25:13 2010
@@ -0,0 +1,252 @@
+% User's Guide
+% Yang Zhang
+
+Design
+======
+
+In Hedwig, clients publish messages associated with a topic, and they subscribe
+to a topic to receive all messages published with that topic.  Clients are
+associated with (publish to and subscribe from) a Hedwig _instance_ (also
+referred to as a _region_), which consists of a number of servers called
+_hubs_.  The hubs partition up topic ownership among themselves, and all
+publishes and subscribes to a topic must be done to its owning hub.  When a
+client doesn't know the owning hub, it tries a default hub, which may redirect
+the client.
+
+Running a Hedwig instance requires a Zookeeper server and at least three
+Bookkeeper servers.
+
+An instance is designed to run within a datacenter.  For wide-area messaging
+across datacenters, specify in the server configuration the set of default
+servers for each of the other instances.  Dissemination among instances
+currently takes place over an all-to-all topology.  Local subscriptions cause
+the hub to subscribe to all other regions on this topic, so that the local
+region receives all updates to it.  Future work includes allowing the user to
+overlay alternative topologies.
+
+Because all messages on a topic go through a single hub per region, all
+messages within a region are ordered.  This means that, for a given topic,
+messages are delivered in the same order to all subscribers within a region,
+and messages from any particular region are delivered in the same order to all
+subscribers globally, but messages from different regions may be delivered in
+different orders to different regions.  Providing global ordering is
+prohibitively expensive in the wide area.  However, in Hedwig clients such as
+PNUTS, the lack of global ordering is not a problem, as PNUTS serializes all
+updates to a table row at a single designated master for that row.
+
+Topics are independent; Hedwig provides no ordering across different topics.
+
+Version vectors are associated with each topic and serve as the identifiers for
+each message.  Vectors consist of one component per region.  A component value
+is the region's local sequence number on the topic, and is incremented each
+time a hub persists a message (published either locally or remotely) to BK.
+
+TODO: More on how version vectors are to be used, and on maintaining
+vector-maxes.
+
+Entry Points
+============
+
+The main class for running the server is
+`org.apache.hedwig.server.netty.PubSubServer`.  It takes a single argument,
+which is a [Commons Configuration] file.  Currently, for configuration, the
+source is the documentation.  See
+`org.apache.hedwig.server.conf.ServerConfiguration` for server configuration
+parameters.
+
+The client is a library intended to be consumed by user applications.  It takes
+a Commons Configuration object, for which the source/documentation is in
+`org.apache.hedwig.client.conf.ClientConfiguration`.
+
+We have provided a simple client application, `org.apache.hedwig.client.App`,
+that can drive a number of benchmarks.  This also takes a single configuration
+file argument, which is fed to the client library.
+
+We've provided a number of scripts to faciliate running servers and clients
+in a variety of configurations, including over distributed hosts.  These are
+all consolidated in `scripts/hw.bash`.  Although the snippets in this
+documentation run the script from the hedwig main directory, you can run it
+from any location.  Apologies in advance for these being bash scripts; time
+permitting, a more robust and maintainable support/tooling infrastructure would
+be ideal.
+
+[Commons Configuration]: http://commons.apache.org/configuration/
+
+Deployment
+==========
+
+When ssh-ing into a new host, you are requested to verify and accept the host
+key.  In order to automatically accept the host keys for many new hosts
+(dangerous), use:
+
+  hosts="$host1 $host2 ..." scripts/hw.bash warmup
+
+The `hosts` variable is set here to the list of hosts that you would like to
+warm up.
+
+To set up JDK6 on some hosts, use:
+
+  hosts="..." scripts/hw.bash setup-java $path_to_modified_jdk6
+
+The argument must point to a JDK6 binary self-extracting executable, but with
+the `more` command that displays the License agreement replaced with
+`cat`.  Unfortunately, this step must be performed manually.  This script will
+extract the JDK directly into the home directory and update `$PATH` in
+`~/.bashrc` (in an idempotent fashion).
+
+Because the current implementation uses a single socket per subscription, the
+Hedwig launching scripts all require a high `ulimit` on the number of open file
+descriptors.  Non-root users can only use up to the limit specified in
+`/etc/security/limits.conf`; to raise this to 1024^2, run:
+
+  hosts="..." scripts/hw.bash setup-limits
+
+This uses `ssh` so that you need to enter your password for `sudo` just
+once.
+
+For most of the commands presented in the next section, you may prefix the
+command with:
+
+  push_jar=true ...
+
+to first push the assembly jar (assumed to be available in `server/target/`) to
+all hosts.
+
+Running Servers
+===============
+
+To start three BK bookies on ports 3181-3183 on localhost (directories must all
+exist):
+
+  scripts/hw.bash bk 3181 $bk1_journal_dir $bk1_ledger_dir &
+  scripts/hw.bash bk 3182 $bk2_journal_dir $bk2_ledger_dir &
+  scripts/hw.bash bk 3183 $bk3_journal_dir $bk3_ledger_dir &
+
+To start a ZK on port 2181 (directory must exist):
+
+  scripts/hw.bash zk 2181 /path/for/zk/ &
+
+To register the BKs with the ZK (so that Hedwig knows where to find the
+bookies):
+
+  scripts/hw.bash setup-bk localhost:2181 `hostname`:3181 `hostname`:3182 `hostname`:3183
+
+Everything up to this point may be done using a single command over a set of
+hosts, with ZK on port 9877 and BK on port 9878. The following function takes
+2 arguments.  The first is the ZK host.  The second is a string list of BK hosts:
+
+  scripts/hw.bash start-zk-bks $zkhost "$bk1host $bk2host $bk3host ..."
+
+Note that the hosts may be SSH profile aliases in your `~/.ssh/config`; the
+script will parse this file and look up their hostnames where necessary.  This
+applies for the hosts specified in the other commands.
+
+Also, the scripts use the `bk-journal` and `bk-ledger` functions in `hw.bash`
+to determine where to place the BK journal and ledger, given a hostname.
+
+To start a Hedwig server locally:
+
+  scripts/hw.bash hw server.conf &
+
+To start Hedwig servers on some hosts "$hw1host $hw2host $hw3host ..." on port 9876, 
+using $zkhost as the ZK server:
+
+  scripts/hw.bash start-hw '' "$hw1host $hw2host $hw3host ..." $zkhost
+
+Above, the first empty string argument is the list of default servers to each 
+of the other regions.  You may run multiple connected instances of Hedwig this way.  
+E.g., to start three regions each with a single Hedwig hub that talk to each other,
+and using the hw.bash default server ports of 9875 (non-SSL) and 9876 (SSL):
+
+  scripts/hw.bash start-hw "$hw2host:9875:9876 $hw3host:9875:9876" "$hw1host" $zk1host
+  scripts/hw.bash start-hw "$hw1host:9875:9876 $hw3host:9875:9876" "$hw2host" $zk2host
+  scripts/hw.bash start-hw "$hw1host:9875:9876 $hw2host:9875:9876" "$hw3host" $zk3host
+
+Everything up to this point may be done using a single command over a set of
+hosts:
+
+  scripts/hw.bash start-region '' "$hw1host $hw2host $hw3host ..." $zkhost "$bk1host $bk2host $bk3host ..."
+
+The first three arguments are the same as for `start-hw`.
+
+You may start multiple regions as well:
+
+  scripts/hw.bash start-regions regions.cfg
+
+"regions.cfg" is a list of all regions, one per line, with each region having 
+the following format:
+
+  region=<Region name>, hub=<list of hub servers>, default=<single hub server>, zk=<single ZK server>, bk=<list of BK servers>
+
+This will create all of the regions with an all-to-all topology. Each region 
+is connected to the default hub server of every other region. The ", " delimiter
+is used to separate out the different parts of a region along with the hard-coded 
+parameter names. There also needs to be a newline after the last region line.
+Here is an example file specifying three regions:
+
+  region=wilbur, hub=wilbur90 wilbur91, default=wilbur90, zk=wilbur93, bk=wilbur93 wilbur94 wilbur95
+  region=re1, hub=sherpa7 sherpa8, default=sherpa7, zk=sherpa9, bk=sherpa9 sherpa10 sherpa11
+  region=peanuts, hub=peanuts1 peanuts2, default=peanuts2, zk=peanuts3, bk=peanuts3 peanuts4 peanuts5
+
+Running the Client
+==================
+
+To run the test client:
+
+  JAVAFLAGS="..." scripts/hw.bash hwc $conf_path
+
+where `$conf_path` is a client configuration file.
+
+To run the test client on some other hosts:
+
+  hosts="..." JAVAFLAGS="..." scripts/hw.bash app $hwhost
+
+This will generate a simple configuration file assuming $hwhost is listening on
+the default SSL and non-SSL ports which are specified as global variables in hw.bash.
+Currently these are 9875 for non-SSL and 9876 for SSL.
+
+Client usage is currently documented in the source.  To run a subscription
+benchmark, set `JAVAFLAGS` to:
+
+  -Dmode=sub -Dcount=10000 -Dnpar=100 -Dstart=5 -Dnwarmups=30
+
+This will first create 30 warm-up subscriptions to topics "warmup-5" through
+"warmup-34", then 10,000 benchmarked subscriptions to topics "topic-5" through
+"topic-10,004".  It will have a pipeline depth of 100 requests, meaning that
+there will be at most 100 outstanding (unresponded) messages in flight at any
+moment.
+
+To run a publishing benchmark, set `JAVAFLAGS` to:
+
+  -Dmode=pub -Dcount=10000 -Dnpar=100 -Dstart=5
+
+This will publish 10,000 messages to topic "topic-5", with a pipeline depth of
+100 requests.
+
+At the end, the programs will print throughput and latency information.
+
+Utilities
+=========
+
+To kill all the user's Java processes running on some machines, use:
+
+  hosts="..." scripts/hw.bash dkill
+
+To check if any processes are running and are using ports of interest (esp.
+9876-9878):
+
+  hosts="..." scripts/hw.bash dstatus
+
+Add an argument to `dstatus` (may be anything) to get a more detailed listing.
+
+To check if there's anything consuming the CPU on some machines:
+
+  hosts="..." scripts/hw.bash tops
+
+To run an arbitrary command on multiple hosts in parallel:
+
+  hosts="..." scripts/hw.bash parssh $command
+
+To do this in sequence:
+
+  hosts="..." xargs= scripts/hw.bash parssh $command



Mime
View raw message