incubator-giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jgho...@apache.org
Subject svn commit: r1242614 - in /incubator/giraph/trunk: ./ src/main/java/org/apache/giraph/graph/ src/main/java/org/apache/giraph/zk/ src/test/java/zk/
Date Fri, 10 Feb 2012 00:13:04 GMT
Author: jghoman
Date: Fri Feb 10 00:13:04 2012
New Revision: 1242614

URL: http://svn.apache.org/viewvc?rev=1242614&view=rev
Log:
GIRAPH-142: _hadoopBsp should be prefixable via configuration.

Added:
    incubator/giraph/trunk/src/test/java/zk/
    incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java
Modified:
    incubator/giraph/trunk/CHANGELOG
    incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java
    incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java
    incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java

Modified: incubator/giraph/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1242614&r1=1242613&r2=1242614&view=diff
==============================================================================
--- incubator/giraph/trunk/CHANGELOG (original)
+++ incubator/giraph/trunk/CHANGELOG Fri Feb 10 00:13:04 2012
@@ -2,6 +2,8 @@ Giraph Change Log
 
 Release 0.2.0 - unreleased
 
+  GIRAPH-142: _hadoopBsp should be prefixable via configuration. (jghoman)
+
   GIRAPH-145. Change partition request log level to debug rather than info.
   (jghoman)
 

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java?rev=1242614&r1=1242613&r2=1242614&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java (original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java Fri Feb 10
00:13:04 2012
@@ -23,6 +23,7 @@ import org.apache.giraph.graph.partition
 import org.apache.giraph.zk.BspEvent;
 import org.apache.giraph.zk.PredicateLock;
 import org.apache.giraph.zk.ZooKeeperExt;
+import org.apache.giraph.zk.ZooKeeperManager;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -137,7 +138,7 @@ public abstract class BspService <
     /** Unset application attempt */
     public static final long UNSET_APPLICATION_ATTEMPT = Long.MIN_VALUE;
 
-    public static final String BASE_DIR = "/_hadoopBsp";
+    private static final String BASE_DIR = "/_hadoopBsp";
     public static final String MASTER_JOB_STATE_NODE = "/_masterJobState";
     public static final String INPUT_SPLIT_DIR = "/_inputSplitDir";
     public static final String INPUT_SPLIT_DONE_DIR = "/_inputSplitDoneDir";
@@ -629,7 +630,7 @@ public abstract class BspService <
             conf.getInt(GiraphJob.CHECKPOINT_FREQUENCY,
                           GiraphJob.CHECKPOINT_FREQUENCY_DEFAULT);
 
-        BASE_PATH = BASE_DIR + "/" + jobId;
+        BASE_PATH = ZooKeeperManager.getBasePath(conf) + BASE_DIR + "/" + jobId;
         MASTER_JOB_STATE_PATH = BASE_PATH + MASTER_JOB_STATE_NODE;
         INPUT_SPLIT_PATH = BASE_PATH + INPUT_SPLIT_DIR;
         INPUT_SPLITS_ALL_READY_PATH = BASE_PATH + INPUT_SPLITS_ALL_READY_NODE;

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java?rev=1242614&r1=1242613&r2=1242614&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java (original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/graph/GiraphJob.java Fri Feb 10
00:13:04 2012
@@ -280,6 +280,12 @@ public class GiraphJob extends Job {
     public static final String RESTART_SUPERSTEP = "giraph.restartSuperstep";
 
     /**
+     * Base ZNode for Giraph's state in the ZooKeeper cluster.  Must be a root
+     * znode on the cluster beginning with "/"
+     */
+    public static final String BASE_ZNODE_KEY = "giraph.zkBaseZNode";
+
+    /**
      * If ZOOKEEPER_LIST is not set, then use this directory to manage
      * ZooKeeper
      */

Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java?rev=1242614&r1=1242613&r2=1242614&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java (original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java Fri Feb
10 00:13:04 2012
@@ -18,6 +18,19 @@
 
 package org.apache.giraph.zk;
 
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.Closeables;
+import org.apache.commons.io.FileUtils;
+import org.apache.giraph.graph.GiraphJob;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileWriter;
@@ -35,18 +48,8 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.Closeables;
-import org.apache.commons.io.FileUtils;
-import org.apache.giraph.graph.GiraphJob;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.log4j.Logger;
-import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import static org.apache.giraph.graph.GiraphJob.BASE_ZNODE_KEY;
+
 
 /**
  * Manages the election of ZooKeeper servers, starting/stopping the services,
@@ -128,6 +131,24 @@ public class ZooKeeperManager {
         return GiraphJob.ZOOKEEPER_MANAGER_DIR_DEFAULT + "/" + jobId;
     }
 
+  /**
+   * Return the base ZooKeeper ZNode from which all other ZNodes Giraph creates
+   * should be sited, for instance in a multi-tenant ZooKeeper, the znode
+   * reserved for Giraph
+   *
+   * @param conf  Necessary to access user-provided values
+   * @return  String of path without trailing slash
+   */
+    public static String getBasePath(Configuration conf) {
+        String result = conf.get(BASE_ZNODE_KEY, "");
+        if(!result.equals("") && !result.startsWith("/")) {
+            throw new IllegalArgumentException("Value for " +
+                BASE_ZNODE_KEY + " must start with /: " + result);
+        }
+
+        return result;
+    }
+
     /**
      * Collects the output of a stream and dumps it to the log.
      */

Added: incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java?rev=1242614&view=auto
==============================================================================
--- incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java (added)
+++ incubator/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java Fri Feb 10 00:13:04
2012
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package zk;
+
+import junit.framework.TestCase;
+import org.apache.giraph.graph.GiraphJob;
+import org.apache.giraph.zk.ZooKeeperManager;
+import org.apache.hadoop.conf.Configuration;
+
+public class TestZooKeeperManager extends TestCase {
+  public void testGetBasePath() {
+    Configuration conf = new Configuration();
+
+    // Default is empty, everything goes in root znode
+    assertEquals("Default value for base path should be empty",
+        "", ZooKeeperManager.getBasePath(conf));
+
+    conf.set(GiraphJob.BASE_ZNODE_KEY, "/howdy");
+    assertEquals("Base path should reflect value of " + GiraphJob.BASE_ZNODE_KEY,
+        "/howdy", ZooKeeperManager.getBasePath(conf));
+
+    conf.set(GiraphJob.BASE_ZNODE_KEY, "no_slash");
+    try {
+      ZooKeeperManager.getBasePath(conf);
+      fail("Should not have allowed path without starting slash");
+    } catch (IllegalArgumentException iae) {
+      assertTrue(iae.getMessage().contains(GiraphJob.BASE_ZNODE_KEY));
+    }
+  }
+}



Mime
View raw message