hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r999057 - in /hbase/trunk: ./ conf/ src/main/java/org/apache/hadoop/hbase/master/ src/main/java/org/apache/hadoop/hbase/regionserver/ src/main/java/org/apache/hadoop/hbase/zookeeper/
Date Mon, 20 Sep 2010 19:04:41 GMT
Author: stack
Date: Mon Sep 20 19:04:41 2010
New Revision: 999057

URL: http://svn.apache.org/viewvc?rev=999057&view=rev
Log:
HBASE-3017 More log pruning

M conf/log4j.properties
  Make ZKW log at INFO-level
M src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
  Log message cleanup.
M src/main/java/org/apache/hadoop/hbase/master/HMaster.java
  Remove excessive hostname+port qualifier on master for zk messages
M src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
  Log message cleanup
M src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
  Format the ServerMonitor message.
M src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
  Remove excessive hostname on zk message id; just add port
M src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
  Cleanup of messages.

Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/conf/log4j.properties
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=999057&r1=999056&r2=999057&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Mon Sep 20 19:04:41 2010
@@ -919,6 +919,7 @@ Release 0.21.0 - Unreleased
    HBASE-2988  Support alternate compression for major compactions
    HBASE-2941  port HADOOP-6713 - threading scalability for RPC reads - to HBase
    HBASE-2782  QOS for META table access
+   HBASE-3017  More log pruning
 
   NEW FEATURES
    HBASE-1961  HBase EC2 scripts

Modified: hbase/trunk/conf/log4j.properties
URL: http://svn.apache.org/viewvc/hbase/trunk/conf/log4j.properties?rev=999057&r1=999056&r2=999057&view=diff
==============================================================================
--- hbase/trunk/conf/log4j.properties (original)
+++ hbase/trunk/conf/log4j.properties Mon Sep 20 19:04:41 2010
@@ -43,6 +43,8 @@ log4j.appender.console.layout.Conversion
 log4j.logger.org.apache.zookeeper=INFO
 #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
 log4j.logger.org.apache.hadoop.hbase=DEBUG
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
 #log4j.logger.org.apache.hadoop.dfs=DEBUG
 # Set this class to log INFO only otherwise its OTT
-log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=999057&r1=999056&r2=999057&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Mon Sep
20 19:04:41 2010
@@ -253,7 +253,7 @@ public class AssignmentManager extends Z
       RegionState regionState = regionsInTransition.get(encodedName);
       switch(data.getEventType()) {
         case M2ZK_REGION_OFFLINE:
-          LOG.warn("What to do with this event? " + data);
+          // Nothing to do.
           break;
 
         case RS2ZK_REGION_CLOSING:
@@ -545,8 +545,8 @@ public class AssignmentManager extends Z
       if (plan == null) {
         LOG.debug("No previous transition plan for " +
             state.getRegion().getRegionNameAsString() +
-            " so generating a random one from " + serverManager.countOfRegionServers() +
-            " ( " + serverManager.getOnlineServers().size() + ") available servers");
+            " so generating a random one; " + serverManager.countOfRegionServers() +
+            " (online=" + serverManager.getOnlineServers().size() + ") available servers");
         plan = new RegionPlan(state.getRegion(), null,
           LoadBalancer.randomAssignment(serverManager.getOnlineServersList()));
         regionPlans.put(encodedName, plan);

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=999057&r1=999056&r2=999057&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Mon Sep 20 19:04:41
2010
@@ -203,8 +203,7 @@ implements HMasterInterface, HMasterRegi
         "_" + System.currentTimeMillis());
     }
 
-    this.zooKeeper =
-      new ZooKeeperWatcher(conf, MASTER + "-" + getMasterAddress(), this);
+    this.zooKeeper = new ZooKeeperWatcher(conf, MASTER, this);
 
     /*
      * 2. Block on becoming the active master.

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=999057&r1=999056&r2=999057&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Mon Sep
20 19:04:41 2010
@@ -165,13 +165,13 @@ public class MasterFileSystem {
     }
     for (FileStatus status : logFolders) {
       String serverName = status.getPath().getName();
-      LOG.info("Found log folder : " + serverName);
-      if(onlineServers.get(serverName) == null) {
-        LOG.info("Log folder doesn't belong " +
+      if (onlineServers.get(serverName) == null) {
+        LOG.info("Log folder " + status.getPath() + " doesn't belong " +
           "to a known region server, splitting");
         splitLog(serverName);
       } else {
-        LOG.info("Log folder belongs to an existing region server");
+        LOG.info("Log folder " + status.getPath() +
+          " belongs to an existing region server");
       }
     }
   }
@@ -279,4 +279,4 @@ public class MasterFileSystem {
         new Path(rootdir, region.getTableDesc().getNameAsString()),
         region.getEncodedName(), familyName), true);
   }
-}
\ No newline at end of file
+}

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=999057&r1=999056&r2=999057&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Mon Sep 20
19:04:41 2010
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.text.DecimalFormat;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -93,6 +94,8 @@ public class ServerManager {
 
   private final DeadServer deadservers = new DeadServer();
 
+  private static final DecimalFormat DF = new DecimalFormat("#.##");
+
   /**
    * Dumps into log current stats on dead servers and number of servers
    * TODO: Make this a metric; dump metrics into log.
@@ -108,10 +111,9 @@ public class ServerManager {
       int numDeadServers = deadservers.size();
       double averageLoad = getAverageLoad();
       String deadServersList = deadservers.toString();
-      LOG.info(numServers + " region servers, " + numDeadServers +
-        " dead, average load " + averageLoad +
-        ((deadServersList != null && deadServersList.length() > 0)?
-          deadServersList: ""));
+      LOG.info("regionservers=" + numServers +
+        ", averageload=" + DF.format(averageLoad) +
+        ((numDeadServers > 0)?  ("deadservers=" + deadServersList): ""));
     }
   }
 
@@ -422,7 +424,7 @@ public class ServerManager {
         LOG.info("Waiting on following regionserver(s) to go down " +
           this.onlineServers.values());
         try {
-          this.onlineServers.wait(500);
+          this.onlineServers.wait(1000);
         } catch (InterruptedException e) {
           // continue
         }
@@ -516,7 +518,7 @@ public class ServerManager {
         HConnectionManager.getConnection(this.master.getConfiguration());
       HRegionInterface hri = serverConnections.get(info.getServerName());
       if (hri == null) {
-        LOG.info("new connection");
+        LOG.debug("New connection to " + info.getServerName());
         hri = connection.getHRegionConnection(info.getServerAddress(), false);
         serverConnections.put(info.getServerName(), hri);
       }
@@ -537,9 +539,10 @@ public class ServerManager {
       getLong("hbase.master.wait.on.regionservers.interval", 3000);
     // So, number of regionservers > 0 and its been n since last check in, break,
     // else just stall here
+    int count = 0;
     for (int oldcount = countOfRegionServers(); !this.master.isStopped();) {
       Thread.sleep(interval);
-      int count = countOfRegionServers();
+      count = countOfRegionServers();
       if (count == oldcount && count > 0) break;
       if (count == 0) {
         LOG.info("Waiting on regionserver(s) to checkin");
@@ -548,6 +551,8 @@ public class ServerManager {
       }
       oldcount = count;
     }
+    LOG.info("Exiting wait on regionserver(s) to checkin; count=" + count +
+      ", stopped=" + this.master.isStopped());
   }
 
   public List<HServerInfo> getOnlineServersList() {

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=999057&r1=999056&r2=999057&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Mon
Sep 20 19:04:41 2010
@@ -431,8 +431,8 @@ public class HRegionServer implements HR
 
   private void initializeZooKeeper() throws IOException, InterruptedException {
     // open connection to zookeeper and set primary watcher
-    zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + "-"
-        + serverInfo.getServerName(), this);
+    zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + ":" +
+      serverInfo.getServerAddress().getPort(), this);
 
     this.clusterStatusTracker = new ClusterStatusTracker(this.zooKeeper, this);
     this.clusterStatusTracker.start();

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java?rev=999057&r1=999056&r2=999057&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java Mon Sep 20 19:04:41
2010
@@ -135,8 +135,8 @@ public class ZKAssign {
   public static void createNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region,
       String serverName, final EventType event)
   throws KeeperException, KeeperException.NodeExistsException {
-    LOG.debug(zkw.prefix("Creating an unassigned node for " +
-      region.getEncodedName() + " in an OFFLINE state"));
+    LOG.debug(zkw.prefix("Creating unassigned node for " +
+      region.getEncodedName() + " in OFFLINE state"));
     RegionTransitionData data = new RegionTransitionData(event,
       region.getRegionName(), serverName);
     synchronized(zkw.getNodes()) {
@@ -167,8 +167,8 @@ public class ZKAssign {
   public static void forceNodeOffline(ZooKeeperWatcher zkw, HRegionInfo region,
       String serverName)
   throws KeeperException, KeeperException.NoNodeException {
-    LOG.debug(zkw.prefix("Forcing an existing unassigned node for " +
-      region.getEncodedName() + " to an OFFLINE state"));
+    LOG.debug(zkw.prefix("Forcing existing unassigned node for " +
+      region.getEncodedName() + " to OFFLINE state"));
     RegionTransitionData data = new RegionTransitionData(
         EventType.M2ZK_REGION_OFFLINE, region.getRegionName(), serverName);
     synchronized(zkw.getNodes()) {
@@ -200,8 +200,8 @@ public class ZKAssign {
   public static boolean createOrForceNodeOffline(ZooKeeperWatcher zkw,
       HRegionInfo region, String serverName)
   throws KeeperException {
-    LOG.debug(zkw.prefix("Creating or updating an unassigned node for " +
-      region.getEncodedName() + " with an OFFLINE state"));
+    LOG.debug(zkw.prefix("Creating (or updating) unassigned node for " +
+      region.getEncodedName() + " with OFFLINE state"));
     RegionTransitionData data = new RegionTransitionData(
         EventType.M2ZK_REGION_OFFLINE, region.getRegionName(), serverName);
     synchronized(zkw.getNodes()) {
@@ -319,7 +319,7 @@ public class ZKAssign {
   private static boolean deleteNode(ZooKeeperWatcher zkw, String regionName,
       EventType expectedState)
   throws KeeperException, KeeperException.NoNodeException {
-    LOG.debug(zkw.prefix("Deleting an existing unassigned " +
+    LOG.debug(zkw.prefix("Deleting existing unassigned " +
       "node for " + regionName + " that is in expected state " + expectedState));
     String node = getNodeName(zkw, regionName);
     Stat stat = new Stat();
@@ -329,7 +329,7 @@ public class ZKAssign {
     }
     RegionTransitionData data = RegionTransitionData.fromBytes(bytes);
     if(!data.getEventType().equals(expectedState)) {
-      LOG.warn(zkw.prefix("Attempting to delete an unassigned " +
+      LOG.warn(zkw.prefix("Attempting to delete unassigned " +
         "node in " + expectedState +
         " state but node is in " + data.getEventType() + " state"));
       return false;
@@ -338,7 +338,7 @@ public class ZKAssign {
       // TODO: Does this go here or only if we successfully delete node?
       zkw.getNodes().remove(node);
       if(!ZKUtil.deleteNode(zkw, node, stat.getVersion())) {
-        LOG.warn(zkw.prefix("Attempting to delete an " +
+        LOG.warn(zkw.prefix("Attempting to delete " +
           "unassigned node in " + expectedState +
             " state but " +
             "after verifying it was in OPENED state, we got a version mismatch"));
@@ -392,7 +392,7 @@ public class ZKAssign {
   public static int createNodeClosing(ZooKeeperWatcher zkw, HRegionInfo region,
       String serverName)
   throws KeeperException, KeeperException.NodeExistsException {
-    LOG.debug(zkw.prefix("Creating an unassigned node for " +
+    LOG.debug(zkw.prefix("Creating unassigned node for " +
       region.getEncodedName() + " in a CLOSING state"));
     RegionTransitionData data = new RegionTransitionData(
         EventType.RS2ZK_REGION_CLOSING, region.getRegionName(), serverName);



Mime
View raw message