hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject svn commit: r1425513 [4/7] - in /hbase/branches/0.94-test: ./ bin/ conf/ security/src/main/java/org/apache/hadoop/hbase/ipc/ security/src/main/java/org/apache/hadoop/hbase/security/access/ security/src/test/java/org/apache/hadoop/hbase/security/access/...
Date Sun, 23 Dec 2012 19:34:56 GMT
Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java Sun Dec 23 19:34:53 2012
@@ -105,6 +105,8 @@ public class ReplicationSource extends T
   private int replicationQueueNbCapacity;
   // Our reader for the current log
   private HLog.Reader reader;
+  // Current position in the log
+  private long position = 0;
   // Last position in the log that we sent to ZooKeeper
   private long lastLoggedPosition = -1;
   // Path of the current log
@@ -130,15 +132,10 @@ public class ReplicationSource extends T
   private int currentNbEntries = 0;
   // Current number of operations (Put/Delete) that we need to replicate
   private int currentNbOperations = 0;
-  // Current size of data we need to replicate
-  private int currentSize = 0;
   // Indicates if this particular source is running
   private volatile boolean running = true;
   // Metrics for this source
   private ReplicationSourceMetrics metrics;
-  // Handle on the log reader helper
-  private ReplicationHLogReaderManager repLogReader;
-
 
   /**
    * Instantiation method used by region servers
@@ -186,7 +183,7 @@ public class ReplicationSource extends T
         this.conf.getLong("replication.source.sleepforretries", 1000);
     this.fs = fs;
     this.metrics = new ReplicationSourceMetrics(peerClusterZnode);
-    this.repLogReader = new ReplicationHLogReaderManager(this.fs, this.conf);
+
     try {
       this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher());
     } catch (KeeperException ke) {
@@ -266,8 +263,8 @@ public class ReplicationSource extends T
     // normally has a position (unless the RS failed between 2 logs)
     if (this.queueRecovered) {
       try {
-        this.repLogReader.setPosition(this.zkHelper.getHLogRepPosition(
-            this.peerClusterZnode, this.queue.peek().getName()));
+        this.position = this.zkHelper.getHLogRepPosition(
+            this.peerClusterZnode, this.queue.peek().getName());
       } catch (KeeperException e) {
         this.terminate("Couldn't get the position of this recovered queue " +
             peerClusterZnode, e);
@@ -325,7 +322,6 @@ public class ReplicationSource extends T
 
       boolean gotIOE = false;
       currentNbEntries = 0;
-      currentSize = 0;
       try {
         if (readAllEntriesToReplicateOrNextFile(currentWALisBeingWrittenTo)) {
           continue;
@@ -361,7 +357,9 @@ public class ReplicationSource extends T
         }
       } finally {
         try {
-          this.repLogReader.closeReader();
+          if (this.reader != null) {
+            this.reader.close();
+          }
         } catch (IOException e) {
           gotIOE = true;
           LOG.warn("Unable to finalize the tailing of a file", e);
@@ -372,10 +370,10 @@ public class ReplicationSource extends T
       // wait a bit and retry.
       // But if we need to stop, don't bother sleeping
       if (this.isActive() && (gotIOE || currentNbEntries == 0)) {
-        if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
+        if (this.lastLoggedPosition != this.position) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
-          this.lastLoggedPosition = this.repLogReader.getPosition();
+              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
+          this.lastLoggedPosition = this.position;
         }
         if (sleepForRetries("Nothing to replicate", sleepMultiplier)) {
           sleepMultiplier++;
@@ -407,9 +405,11 @@ public class ReplicationSource extends T
   protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo)
       throws IOException{
     long seenEntries = 0;
-    this.repLogReader.seek();
-    HLog.Entry entry =
-        this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
+    if (this.position != 0) {
+      this.reader.seek(this.position);
+    }
+    long startPosition = this.position;
+    HLog.Entry entry = readNextAndSetPosition();
     while (entry != null) {
       WALEdit edit = entry.getEdit();
       this.metrics.logEditsReadRate.inc(1);
@@ -433,18 +433,18 @@ public class ReplicationSource extends T
           }
           currentNbOperations += countDistinctRowKeys(edit);
           currentNbEntries++;
-          currentSize += entry.getEdit().heapSize();
         } else {
           this.metrics.logEditsFilteredRate.inc(1);
         }
       }
       // Stop if too many entries or too big
-      if (currentSize >= this.replicationQueueSizeCapacity ||
+      if ((this.reader.getPosition() - startPosition)
+          >= this.replicationQueueSizeCapacity ||
           currentNbEntries >= this.replicationQueueNbCapacity) {
         break;
       }
       try {
-        entry = this.repLogReader.readNextAndSetPosition(this.entriesArray, this.currentNbEntries);
+        entry = readNextAndSetPosition();
       } catch (IOException ie) {
         LOG.debug("Break on IOE: " + ie.getMessage());
         break;
@@ -452,7 +452,7 @@ public class ReplicationSource extends T
     }
     LOG.debug("currentNbOperations:" + currentNbOperations +
         " and seenEntries:" + seenEntries +
-        " and size: " + this.currentSize);
+        " and size: " + (this.reader.getPosition() - startPosition));
     if (currentWALisBeingWrittenTo) {
       return false;
     }
@@ -461,6 +461,16 @@ public class ReplicationSource extends T
     return seenEntries == 0 && processEndOfFile();
   }
 
+  private HLog.Entry readNextAndSetPosition() throws IOException {
+    HLog.Entry entry = this.reader.next(entriesArray[currentNbEntries]);
+    // Store the position so that in the future the reader can start
+    // reading from here. If the above call to next() throws an
+    // exception, the position won't be changed and retry will happen
+    // from the last known good position
+    this.position = this.reader.getPosition();
+    return entry;
+  } 
+
   private void connectToPeers() {
     // Connect to peer cluster first, unless we have to stop
     while (this.isActive() && this.currentPeers.size() == 0) {
@@ -499,9 +509,10 @@ public class ReplicationSource extends T
   protected boolean openReader(int sleepMultiplier) {
     try {
       LOG.debug("Opening log for replication " + this.currentPath.getName() +
-          " at " + this.repLogReader.getPosition());
+          " at " + this.position);
       try {
-        this.reader = repLogReader.openReader(this.currentPath);
+       this.reader = null;
+       this.reader = HLog.getReader(this.fs, this.currentPath, this.conf);
       } catch (FileNotFoundException fnfe) {
         if (this.queueRecovered) {
           // We didn't find the log in the archive directory, look if it still
@@ -637,10 +648,10 @@ public class ReplicationSource extends T
         HRegionInterface rrs = getRS();
         LOG.debug("Replicating " + currentNbEntries);
         rrs.replicateLogEntries(Arrays.copyOf(this.entriesArray, currentNbEntries));
-        if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
+        if (this.lastLoggedPosition != this.position) {
           this.manager.logPositionAndCleanOldLogs(this.currentPath,
-              this.peerClusterZnode, this.repLogReader.getPosition(), queueRecovered, currentWALisBeingWrittenTo);
-          this.lastLoggedPosition = this.repLogReader.getPosition();
+              this.peerClusterZnode, this.position, queueRecovered, currentWALisBeingWrittenTo);
+          this.lastLoggedPosition = this.position;
         }
         this.totalReplicatedEdits += currentNbEntries;
         this.metrics.shippedBatchesRate.inc(1);
@@ -710,8 +721,7 @@ public class ReplicationSource extends T
   protected boolean processEndOfFile() {
     if (this.queue.size() != 0) {
       this.currentPath = null;
-      this.repLogReader.finishCurrentFile();
-      this.reader = null;
+      this.position = 0;
       return true;
     } else if (this.queueRecovered) {
       this.manager.closeRecoveredQueue(this);

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java Sun Dec 23 19:34:53 2012
@@ -72,6 +72,7 @@ public class RemoteHTable implements HTa
   final Client client;
   final Configuration conf;
   final byte[] name;
+  final String accessToken;
   final int maxRetries;
   final long sleepTime;
 
@@ -80,6 +81,10 @@ public class RemoteHTable implements HTa
       final long startTime, final long endTime, final int maxVersions) {
     StringBuffer sb = new StringBuffer();
     sb.append('/');
+    if (accessToken != null) {
+      sb.append(accessToken);
+      sb.append('/');
+    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append(Bytes.toStringBinary(row));
@@ -137,29 +142,6 @@ public class RemoteHTable implements HTa
     return sb.toString();
   }
 
-  protected String buildMultiRowSpec(final byte[][] rows, int maxVersions) {
-    StringBuilder sb = new StringBuilder();
-    sb.append('/');
-    sb.append(Bytes.toStringBinary(name));
-    sb.append("/multiget/");
-    if (rows == null || rows.length == 0) {
-      return sb.toString();
-    }
-    sb.append("?");
-    for(int i=0; i<rows.length; i++) {
-      byte[] rk = rows[i];
-      if (i != 0) {
-        sb.append('&');
-      }
-      sb.append("row=");
-      sb.append(Bytes.toStringBinary(rk));
-    }
-    sb.append("&v=");
-    sb.append(maxVersions);
-
-    return sb.toString();
-  }
-
   protected Result[] buildResultFromModel(final CellSetModel model) {
     List<Result> results = new ArrayList<Result>();
     for (RowModel row: model.getRows()) {
@@ -205,9 +187,7 @@ public class RemoteHTable implements HTa
    * @param client
    * @param name
    * @param accessToken
-   * @deprecated accessToken is not used and will be removed
    */
-  @Deprecated
   public RemoteHTable(Client client, String name, String accessToken) {
     this(client, HBaseConfiguration.create(), Bytes.toBytes(name), accessToken);
   }
@@ -217,20 +197,8 @@ public class RemoteHTable implements HTa
    * @param client
    * @param conf
    * @param name
-   */
-  public RemoteHTable(Client client, Configuration conf, String name) {
-    this(client, conf, Bytes.toBytes(name), null);
-  }
-
-  /**
-   * Constructor
-   * @param client
-   * @param conf
-   * @param name
    * @param accessToken
-   * @deprecated accessToken is not used and will be removed
    */
-  @Deprecated
   public RemoteHTable(Client client, Configuration conf, String name,
       String accessToken) {
     this(client, conf, Bytes.toBytes(name), accessToken);
@@ -238,28 +206,14 @@ public class RemoteHTable implements HTa
 
   /**
    * Constructor
-   * @param client
-   * @param conf
-   * @param name
-   */
-  public RemoteHTable(Client client, Configuration conf, byte[] name) {
-    this(client, conf, name, null);
-  }
-
-  /**
-   * Constructor
-   * @param client
    * @param conf
-   * @param name
-   * @param accessToken
-   * @deprecated accessToken is not used and will be removed
    */
-  @Deprecated
   public RemoteHTable(Client client, Configuration conf, byte[] name,
       String accessToken) {
     this.client = client;
     this.conf = conf;
     this.name = name;
+    this.accessToken = accessToken;
     this.maxRetries = conf.getInt("hbase.rest.client.max.retries", 10);
     this.sleepTime = conf.getLong("hbase.rest.client.sleep", 1000);
   }
@@ -275,6 +229,10 @@ public class RemoteHTable implements HTa
   public HTableDescriptor getTableDescriptor() throws IOException {
     StringBuilder sb = new StringBuilder();
     sb.append('/');
+    if (accessToken != null) {
+      sb.append(accessToken);
+      sb.append('/');
+    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append("schema");
@@ -309,68 +267,30 @@ public class RemoteHTable implements HTa
     if (get.getFilter() != null) {
       LOG.warn("filters not supported on gets");
     }
-    Result[] results = getResults(spec);
-    if (results.length > 0) {
-      if (results.length > 1) {
-        LOG.warn("too many results for get (" + results.length + ")");
-      }
-      return results[0];
-    } else {
-      return new Result();
-    }
-  }
-
-  public Result[] get(List<Get> gets) throws IOException {
-    byte[][] rows = new byte[gets.size()][];
-    int maxVersions = 1;
-    int count = 0;
-
-    for (Get g : gets) {
-
-      if (count == 0) {
-        maxVersions = g.getMaxVersions();
-      } else if (g.getMaxVersions() != maxVersions) {
-        LOG.warn("MaxVersions on Gets do not match, using the first in the list ("
-            + maxVersions +")");
-      }
-
-      if (g.getFilter() != null) {
-        LOG.warn("filters not supported on gets");
-      }
-
-      rows[count] = g.getRow();
-      count++;
-    }
-
-    String spec = buildMultiRowSpec(rows, maxVersions);
-
-    return getResults(spec);
-  }
-
-  private Result[] getResults(String spec) throws IOException {
     for (int i = 0; i < maxRetries; i++) {
       Response response = client.get(spec, Constants.MIMETYPE_PROTOBUF);
       int code = response.getCode();
       switch (code) {
-        case 200:
-          CellSetModel model = new CellSetModel();
-          model.getObjectFromMessage(response.getBody());
-          Result[] results = buildResultFromModel(model);
-          if (results.length > 0) {
-            return results;
-          }
-          // fall through
-        case 404:
-          return new Result[0];
-
-        case 509:
-          try {
-            Thread.sleep(sleepTime);
-          } catch (InterruptedException e) {
+      case 200:
+        CellSetModel model = new CellSetModel();
+        model.getObjectFromMessage(response.getBody());
+        Result[] results = buildResultFromModel(model);
+        if (results.length > 0) {
+          if (results.length > 1) {
+            LOG.warn("too many results for get (" + results.length + ")");
           }
-          break;
-        default:
-          throw new IOException("get request returned " + code);
+          return results[0];
+        }
+        // fall through
+      case 404:
+        return new Result();
+      case 509:
+        try {
+          Thread.sleep(sleepTime);
+        } catch (InterruptedException e) { }
+        break;
+      default:
+        throw new IOException("get request returned " + code);
       }
     }
     throw new IOException("get request timed out");
@@ -386,6 +306,10 @@ public class RemoteHTable implements HTa
     CellSetModel model = buildModelFromPut(put);
     StringBuilder sb = new StringBuilder();
     sb.append('/');
+    if (accessToken != null) {
+      sb.append(accessToken);
+      sb.append('/');      
+    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append(Bytes.toStringBinary(put.getRow()));
@@ -440,6 +364,10 @@ public class RemoteHTable implements HTa
     // build path for multiput
     StringBuilder sb = new StringBuilder();
     sb.append('/');
+    if (accessToken != null) {
+      sb.append(accessToken);
+      sb.append('/');      
+    }
     sb.append(Bytes.toStringBinary(name));
     sb.append("/$multiput"); // can be any nonexistent row
     for (int i = 0; i < maxRetries; i++) {
@@ -505,6 +433,10 @@ public class RemoteHTable implements HTa
       }
       StringBuffer sb = new StringBuffer();
       sb.append('/');
+      if (accessToken != null) {
+        sb.append(accessToken);
+        sb.append('/');
+      }
       sb.append(Bytes.toStringBinary(name));
       sb.append('/');
       sb.append("scanner");
@@ -643,16 +575,10 @@ public class RemoteHTable implements HTa
     throw new IOException("getRowOrBefore not supported");
   }
 
-  /**
-   * @deprecated {@link RowLock} and associated operations are deprecated
-   */
   public RowLock lockRow(byte[] row) throws IOException {
     throw new IOException("lockRow not implemented");
   }
 
-  /**
-   * @deprecated {@link RowLock} and associated operations are deprecated
-   */
   public void unlockRow(RowLock rl) throws IOException {
     throw new IOException("unlockRow not implemented");
   }
@@ -665,6 +591,10 @@ public class RemoteHTable implements HTa
     CellSetModel model = buildModelFromPut(put);
     StringBuilder sb = new StringBuilder();
     sb.append('/');
+    if (accessToken != null) {
+      sb.append(accessToken);
+      sb.append('/');
+    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append(Bytes.toStringBinary(put.getRow()));
@@ -700,6 +630,10 @@ public class RemoteHTable implements HTa
     CellSetModel model = buildModelFromPut(put);
     StringBuilder sb = new StringBuilder();
     sb.append('/');
+    if (accessToken != null) {
+      sb.append(accessToken);
+      sb.append('/');
+    }
     sb.append(Bytes.toStringBinary(name));
     sb.append('/');
     sb.append(Bytes.toStringBinary(row));
@@ -756,6 +690,11 @@ public class RemoteHTable implements HTa
   }
 
   @Override
+  public Result[] get(List<Get> gets) throws IOException {
+    throw new IOException("get(List<Get>) not supported");
+  }
+
+  @Override
   public <T extends CoprocessorProtocol> T coprocessorProxy(Class<T> protocol,
       byte[] row) {
     throw new

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/security/User.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/security/User.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/security/User.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/security/User.java Sun Dec 23 19:34:53 2012
@@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.security
 
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.util.Methods;
 import org.apache.hadoop.mapred.JobConf;
@@ -217,15 +216,12 @@ public abstract class User {
   }
 
   /**
-   * Returns whether or not secure authentication is enabled for HBase.  Note that
-   * HBase security requires HDFS security to provide any guarantees, so this requires that
-   * both <code>hbase.security.authentication</code> and <code>hadoop.security.authentication</code>
-   * are set to <code>kerberos</code>.
+   * Returns whether or not secure authentication is enabled for HBase
+   * (whether <code>hbase.security.authentication</code> is set to
+   * <code>kerberos</code>.
    */
   public static boolean isHBaseSecurityEnabled(Configuration conf) {
-    return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY)) &&
-        "kerberos".equalsIgnoreCase(
-            conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION));
+    return "kerberos".equalsIgnoreCase(conf.get(HBASE_SECURITY_CONF_KEY));
   }
 
   /* Concrete implementations */

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java Sun Dec 23 19:34:53 2012
@@ -63,7 +63,7 @@ public abstract class AbstractHBaseTool 
   protected abstract void processOptions(CommandLine cmd);
 
   /** The "main function" of the tool */
-  protected abstract int doWork() throws Exception;
+  protected abstract void doWork() throws Exception;
 
   @Override
   public Configuration getConf() {
@@ -99,14 +99,13 @@ public abstract class AbstractHBaseTool 
 
     processOptions(cmd);
 
-    int ret = EXIT_FAILURE;
     try {
-      ret = doWork();
+      doWork();
     } catch (Exception e) {
       LOG.error("Error running command-line tool", e);
       return EXIT_FAILURE;
     }
-    return ret;
+    return EXIT_SUCCESS;
   }
 
   private boolean sanityCheckOptions(CommandLine cmd) {

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/ChecksumType.java Sun Dec 23 19:34:53 2012
@@ -25,6 +25,9 @@ import java.util.zip.Checksum;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.ChecksumFactory;
+
 /**
  * Checksum types. The Checksum type is a one byte number
  * that stores a representation of the checksum algorithm
@@ -67,7 +70,7 @@ public enum ChecksumType {
         ctor = ChecksumFactory.newConstructor(PURECRC32);
         LOG.info("Checksum using " + PURECRC32);
       } catch (Exception e) {
-        LOG.trace(PURECRC32 + " not available.");
+        LOG.info(PURECRC32 + " not available.");
       }
       try {
         // The default checksum class name is java.util.zip.CRC32. 
@@ -77,7 +80,7 @@ public enum ChecksumType {
           LOG.info("Checksum can use " + JDKCRC);
         }
       } catch (Exception e) {
-        LOG.trace(JDKCRC + " not available.");
+        LOG.warn(JDKCRC + " not available. ",  e);
       }
     }
 
@@ -110,7 +113,7 @@ public enum ChecksumType {
         ctor = ChecksumFactory.newConstructor(PURECRC32C);
         LOG.info("Checksum can use " + PURECRC32C);
       } catch (Exception e) {
-        LOG.trace(PURECRC32C + " not available.");
+        LOG.info(PURECRC32C + " not available. ");
       }
     }
 

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Sun Dec 23 19:34:53 2012
@@ -151,7 +151,7 @@ public abstract class FSUtils {
    */
   public static FSDataOutputStream create(FileSystem fs, Path path,
       FsPermission perm, boolean overwrite) throws IOException {
-    LOG.debug("Creating file=" + path + " with permission=" + perm);
+    LOG.debug("Creating file:" + path + "with permission:" + perm);
 
     return fs.create(path, perm, overwrite,
         fs.getConf().getInt("io.file.buffer.size", 4096),
@@ -1013,25 +1013,6 @@ public abstract class FSUtils {
   }
 
   /**
-   * Given a particular region dir, return all the familydirs inside it
-   *
-   * @param fs A file system for the Path
-   * @param regionDir Path to a specific region directory
-   * @return List of paths to valid family directories in region dir.
-   * @throws IOException
-   */
-  public static List<Path> getFamilyDirs(final FileSystem fs, final Path regionDir) throws IOException {
-    // assumes we are in a region dir.
-    FileStatus[] fds = fs.listStatus(regionDir, new FamilyDirFilter(fs));
-    List<Path> familyDirs = new ArrayList<Path>(fds.length);
-    for (FileStatus fdfs: fds) {
-      Path fdPath = fdfs.getPath();
-      familyDirs.add(fdPath);
-    }
-    return familyDirs;
-  }
-
-  /**
    * Filter for HFiles that excludes reference files.
    */
   public static class HFileFilter implements PathFilter {

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Sun Dec 23 19:34:53 2012
@@ -46,7 +46,6 @@ import java.util.concurrent.atomic.Atomi
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -83,7 +82,6 @@ import org.apache.hadoop.hbase.io.hfile.
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
@@ -95,9 +93,6 @@ import org.apache.hadoop.hbase.zookeeper
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
 import org.apache.zookeeper.KeeperException;
 
 import com.google.common.base.Joiner;
@@ -151,7 +146,7 @@ import com.google.common.collect.TreeMul
  * can be used to limit the kinds of repairs hbck will do.  See the code in
  * {@link #printUsageAndExit()} for more details.
  */
-public class HBaseFsck extends Configured implements Tool {
+public class HBaseFsck {
   public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
   public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
   private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
@@ -164,6 +159,7 @@ public class HBaseFsck extends Configure
    * Internal resources
    **********************/
   private static final Log LOG = LogFactory.getLog(HBaseFsck.class.getName());
+  private Configuration conf;
   private ClusterStatus status;
   private HConnection connection;
   private HBaseAdmin admin;
@@ -180,14 +176,12 @@ public class HBaseFsck extends Configure
   private long timelag = DEFAULT_TIME_LAG; // tables whose modtime is older
   private boolean fixAssignments = false; // fix assignment errors?
   private boolean fixMeta = false; // fix meta errors?
-  private boolean checkHdfs = true; // load and check fs consistency?
   private boolean fixHdfsHoles = false; // fix fs holes?
   private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky)
   private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
   private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo)
   private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
   private boolean fixSplitParents = false; // fix lingering split parents
-  private boolean fixReferenceFiles = false; // fix lingering reference store file
 
   // limit checking/fixes to listed tables, if empty attempt to check/fix all
   // -ROOT- and .META. are always checked
@@ -205,7 +199,7 @@ public class HBaseFsck extends Configure
   /*********
    * State
    *********/
-  final private ErrorReporter errors;
+  private ErrorReporter errors = new PrintingErrorReporter();
   int fixes = 0;
 
   /**
@@ -246,9 +240,8 @@ public class HBaseFsck extends Configure
    * @throws ZooKeeperConnectionException if unable to connect to ZooKeeper
    */
   public HBaseFsck(Configuration conf) throws MasterNotRunningException,
-      ZooKeeperConnectionException, IOException, ClassNotFoundException {
-    super(conf);
-    errors = getErrorReporter(conf);
+      ZooKeeperConnectionException, IOException {
+    this.conf = conf;
 
     int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
     executor = new ScheduledThreadPoolExecutor(numThreads);
@@ -265,9 +258,8 @@ public class HBaseFsck extends Configure
    *           if unable to connect to ZooKeeper
    */
   public HBaseFsck(Configuration conf, ExecutorService exec) throws MasterNotRunningException,
-      ZooKeeperConnectionException, IOException, ClassNotFoundException {
-    super(conf);
-    errors = getErrorReporter(getConf());
+      ZooKeeperConnectionException, IOException {
+    this.conf = conf;
     this.executor = exec;
   }
 
@@ -276,8 +268,8 @@ public class HBaseFsck extends Configure
    * online state.
    */
   public void connect() throws IOException {
-    admin = new HBaseAdmin(getConf());
-    meta = new HTable(getConf(), HConstants.META_TABLE_NAME);
+    admin = new HBaseAdmin(conf);
+    meta = new HTable(conf, HConstants.META_TABLE_NAME);
     status = admin.getMaster().getClusterStatus();
     connection = admin.getConnection();
   }
@@ -341,11 +333,11 @@ public class HBaseFsck extends Configure
    */
   public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException {
     // Initial pass to fix orphans.
-    if (shouldCheckHdfs() && (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
-        || shouldFixHdfsOverlaps() || shouldFixTableOrphans())) {
+    if (shouldFixHdfsOrphans() || shouldFixHdfsHoles()
+        || shouldFixHdfsOverlaps() || shouldFixTableOrphans()) {
       LOG.info("Loading regioninfos HDFS");
       // if nothing is happening this should always complete in two iterations.
-      int maxIterations = getConf().getInt("hbase.hbck.integrityrepair.iterations.max", 3);
+      int maxIterations = conf.getInt("hbase.hbck.integrityrepair.iterations.max", 3);
       int curIter = 0;
       do {
         clearState(); // clears hbck state and reset fixes to 0 and.
@@ -399,10 +391,8 @@ public class HBaseFsck extends Configure
     loadDeployedRegions();
 
     // load regiondirs and regioninfos from HDFS
-    if (shouldCheckHdfs()) {
-      loadHdfsRegionDirs();
-      loadHdfsRegionInfos();
-    }
+    loadHdfsRegionDirs();
+    loadHdfsRegionInfos();
 
     // Empty cells in .META.?
     reportEmptyMetaCells();
@@ -439,8 +429,6 @@ public class HBaseFsck extends Configure
       admin.setBalancerRunning(oldBalancer, false);
     }
 
-    offlineReferenceFileRepair();
-
     // Print table summary
     printTableSummary(tablesInfo);
     return errors.summarize();
@@ -467,7 +455,7 @@ public class HBaseFsck extends Configure
    */
   private void adoptHdfsOrphan(HbckInfo hi) throws IOException {
     Path p = hi.getHdfsRegionDir();
-    FileSystem fs = p.getFileSystem(getConf());
+    FileSystem fs = p.getFileSystem(conf);
     FileStatus[] dirs = fs.listStatus(p);
     if (dirs == null) {
       LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " +
@@ -492,7 +480,7 @@ public class HBaseFsck extends Configure
         byte[] start, end;
         HFile.Reader hf = null;
         try {
-          CacheConfig cacheConf = new CacheConfig(getConf());
+          CacheConfig cacheConf = new CacheConfig(conf);
           hf = HFile.createReader(fs, hfile.getPath(), cacheConf);
           hf.loadFileInfo();
           KeyValue startKv = KeyValue.createKeyValueFromKey(hf.getFirstKey());
@@ -540,7 +528,7 @@ public class HBaseFsck extends Configure
     // create new region on hdfs.  move data into place.
     HRegionInfo hri = new HRegionInfo(template.getName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond());
     LOG.info("Creating new region : " + hri);
-    HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template);
+    HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, hri, template);
     Path target = region.getRegionDir();
 
     // rename all the data to new region
@@ -597,67 +585,6 @@ public class HBaseFsck extends Configure
   }
 
   /**
-   * Scan all the store file names to find any lingering reference files,
-   * which refer to some none-exiting files. If "fix" option is enabled,
-   * any lingering reference file will be sidelined if found.
-   * <p>
-   * Lingering reference file prevents a region from opening. It has to
-   * be fixed before a cluster can start properly.
-   */
-  private void offlineReferenceFileRepair() throws IOException {
-    Configuration conf = getConf();
-    Path hbaseRoot = FSUtils.getRootDir(conf);
-    FileSystem fs = hbaseRoot.getFileSystem(conf);
-    Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot);
-    for (Path path: allFiles.values()) {
-      boolean isReference = false;
-      try {
-        isReference = StoreFile.isReference(path);
-      } catch (Throwable t) {
-        // Ignore. Some files may not be store files at all.
-        // For example, files under .oldlogs folder in .META.
-        // Warning message is already logged by
-        // StoreFile#isReference.
-      }
-      if (!isReference) continue;
-
-      Path referredToFile = StoreFile.getReferredToFile(path);
-      if (fs.exists(referredToFile)) continue;  // good, expected
-
-      // Found a lingering reference file
-      errors.reportError(ERROR_CODE.LINGERING_REFERENCE_HFILE,
-        "Found lingering reference file " + path);
-      if (!shouldFixReferenceFiles()) continue;
-
-      // Now, trying to fix it since requested
-      boolean success = false;
-      String pathStr = path.toString();
-
-      // A reference file path should be like
-      // ${hbase.rootdir}/table_name/region_id/family_name/referred_file.region_name
-      // Up 3 directories to get the table folder.
-      // So the file will be sidelined to a similar folder structure.
-      int index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR);
-      for (int i = 0; index > 0 && i < 3; i++) {
-        index = pathStr.lastIndexOf(Path.SEPARATOR_CHAR, index);
-      }
-      if (index > 0) {
-        Path rootDir = getSidelineDir();
-        Path dst = new Path(rootDir, pathStr.substring(index));
-        fs.mkdirs(dst.getParent());
-        LOG.info("Trying to sildeline reference file"
-          + path + " to " + dst);
-        setShouldRerun();
-
-        success = fs.rename(path, dst);
-      }
-      if (!success) {
-        LOG.error("Failed to sideline reference file " + path);
-      }
-    }
-  }
-
-  /**
    * TODO -- need to add tests for this.
    */
   private void reportEmptyMetaCells() {
@@ -713,7 +640,7 @@ public class HBaseFsck extends Configure
     }
 
     Path regioninfo = new Path(regionDir, HRegion.REGIONINFO_FILE);
-    FileSystem fs = regioninfo.getFileSystem(getConf());
+    FileSystem fs = regioninfo.getFileSystem(conf);
 
     FSDataInputStream in = fs.open(regioninfo);
     HRegionInfo hri = new HRegionInfo();
@@ -788,11 +715,11 @@ public class HBaseFsck extends Configure
       if (modTInfo == null) {
         // only executed once per table.
         modTInfo = new TableInfo(tableName);
-        Path hbaseRoot = FSUtils.getRootDir(getConf());
+        Path hbaseRoot = FSUtils.getRootDir(conf);
         tablesInfo.put(tableName, modTInfo);
         try {
           HTableDescriptor htd =
-              FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(getConf()),
+              FSTableDescriptors.getTableDescriptor(hbaseRoot.getFileSystem(conf),
               hbaseRoot, tableName);
           modTInfo.htds.add(htd);
         } catch (IOException ioe) {
@@ -821,7 +748,7 @@ public class HBaseFsck extends Configure
    */
   private Set<String> getColumnFamilyList(Set<String> columns, HbckInfo hbi) throws IOException {
     Path regionDir = hbi.getHdfsRegionDir();
-    FileSystem fs = regionDir.getFileSystem(getConf());
+    FileSystem fs = regionDir.getFileSystem(conf);
     FileStatus[] subDirs = fs.listStatus(regionDir, new FSUtils.FamilyDirFilter(fs));
     for (FileStatus subdir : subDirs) {
       String columnfamily = subdir.getPath().getName();
@@ -844,7 +771,7 @@ public class HBaseFsck extends Configure
     for (String columnfamimly : columns) {
       htd.addFamily(new HColumnDescriptor(columnfamimly));
     }
-    FSTableDescriptors.createTableDescriptor(htd, getConf(), true);
+    FSTableDescriptors.createTableDescriptor(htd, conf, true);
     return true;
   }
   
@@ -860,12 +787,12 @@ public class HBaseFsck extends Configure
   public void fixOrphanTables() throws IOException {
     if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) {
 
-      Path hbaseRoot = FSUtils.getRootDir(getConf());
+      Path hbaseRoot = FSUtils.getRootDir(conf);
       List<String> tmpList = new ArrayList<String>();
       tmpList.addAll(orphanTableDirs.keySet());
       HTableDescriptor[] htds = getHTableDescriptors(tmpList);
-      Iterator<Entry<String, Set<String>>> iter = orphanTableDirs.entrySet().iterator();
-      int j = 0;
+      Iterator iter = orphanTableDirs.entrySet().iterator();
+      int j = 0; 
       int numFailedCase = 0;
       while (iter.hasNext()) {
         Entry<String, Set<String>> entry = (Entry<String, Set<String>>) iter.next();
@@ -876,7 +803,7 @@ public class HBaseFsck extends Configure
             HTableDescriptor htd = htds[j];
             LOG.info("fixing orphan table: " + tableName + " from cache");
             FSTableDescriptors.createTableDescriptor(
-                hbaseRoot.getFileSystem(getConf()), hbaseRoot, htd, true);
+                hbaseRoot.getFileSystem(conf), hbaseRoot, htd, true);
             j++;
             iter.remove();
           }
@@ -915,8 +842,8 @@ public class HBaseFsck extends Configure
    * @return an open .META. HRegion
    */
   private HRegion createNewRootAndMeta() throws IOException {
-    Path rootdir = new Path(getConf().get(HConstants.HBASE_DIR));
-    Configuration c = getConf();
+    Path rootdir = new Path(conf.get(HConstants.HBASE_DIR));
+    Configuration c = conf;
     HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO);
     MasterFileSystem.setInfoFamilyCachingForRoot(false);
     HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
@@ -1059,7 +986,7 @@ public class HBaseFsck extends Configure
     for (TableInfo tInfo : tablesInfo.values()) {
       TableIntegrityErrorHandler handler;
       if (fixHoles || fixOverlaps) {
-        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, getConf(),
+        handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
           fixHoles, fixOverlaps);
       } else {
         handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
@@ -1074,7 +1001,7 @@ public class HBaseFsck extends Configure
 
   private Path getSidelineDir() throws IOException {
     if (sidelineDir == null) {
-      Path hbaseDir = FSUtils.getRootDir(getConf());
+      Path hbaseDir = FSUtils.getRootDir(conf);
       Path hbckDir = new Path(hbaseDir, HConstants.HBCK_SIDELINEDIR_NAME);
       sidelineDir = new Path(hbckDir, hbaseDir.getName() + "-"
           + startMillis);
@@ -1191,8 +1118,8 @@ public class HBaseFsck extends Configure
    */
   Path sidelineOldRootAndMeta() throws IOException {
     // put current -ROOT- and .META. aside.
-    Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
-    FileSystem fs = hbaseDir.getFileSystem(getConf());
+    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = hbaseDir.getFileSystem(conf);
     Path backupDir = getSidelineDir();
     fs.mkdirs(backupDir);
 
@@ -1224,7 +1151,7 @@ public class HBaseFsck extends Configure
    */
   private void loadDisabledTables()
   throws ZooKeeperConnectionException, IOException {
-    HConnectionManager.execute(new HConnectable<Void>(getConf()) {
+    HConnectionManager.execute(new HConnectable<Void>(conf) {
       @Override
       public Void connect(HConnection connection) throws IOException {
         ZooKeeperWatcher zkw = connection.getZooKeeperWatcher();
@@ -1252,8 +1179,8 @@ public class HBaseFsck extends Configure
    * regionInfoMap
    */
   public void loadHdfsRegionDirs() throws IOException, InterruptedException {
-    Path rootDir = new Path(getConf().get(HConstants.HBASE_DIR));
-    FileSystem fs = rootDir.getFileSystem(getConf());
+    Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = rootDir.getFileSystem(conf);
 
     // list all tables from HDFS
     List<FileStatus> tableDirs = Lists.newArrayList();
@@ -1281,8 +1208,8 @@ public class HBaseFsck extends Configure
         LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME
             + " file.");
         setShouldRerun();
-        FSUtils.setVersion(fs, rootDir, getConf().getInt(
-            HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), getConf().getInt(
+        FSUtils.setVersion(fs, rootDir, conf.getInt(
+            HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000), conf.getInt(
             HConstants.VERSION_FILE_WRITE_ATTEMPTS,
             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
       }
@@ -1408,8 +1335,8 @@ public class HBaseFsck extends Configure
       return;
     }
 
-    Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR));
-    FileSystem fs = hbaseDir.getFileSystem(getConf());
+    Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
+    FileSystem fs = hbaseDir.getFileSystem(conf);
     UserGroupInformation ugi = User.getCurrent().getUGI();
     FileStatus[] files = fs.listStatus(hbaseDir);
     for (FileStatus file : files) {
@@ -1565,12 +1492,8 @@ public class HBaseFsck extends Configure
       errors.print(msg);
       undeployRegions(hbi);
       setShouldRerun();
-      HRegionInfo hri = hbi.getHdfsHRI();
-      if (hri == null) {
-        hri = hbi.metaEntry;
-      }
-      HBaseFsckRepair.fixUnassigned(admin, hri);
-      HBaseFsckRepair.waitUntilAssigned(admin, hri);
+      HBaseFsckRepair.fixUnassigned(admin, hbi.getHdfsHRI());
+      HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
     }
   }
 
@@ -1582,8 +1505,7 @@ public class HBaseFsck extends Configure
     String descriptiveName = hbi.toString();
 
     boolean inMeta = hbi.metaEntry != null;
-    // In case not checking HDFS, assume the region is on HDFS
-    boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
+    boolean inHdfs = hbi.getHdfsRegionDir()!= null;
     boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
     boolean isDeployed = !hbi.deployedOn.isEmpty();
     boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
@@ -1593,7 +1515,7 @@ public class HBaseFsck extends Configure
     boolean splitParent =
       (hbi.metaEntry == null)? false: hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
     boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry);
-    boolean recentlyModified = inHdfs &&
+    boolean recentlyModified = hbi.getHdfsRegionDir() != null &&
       hbi.getModTime() + timelag > System.currentTimeMillis();
 
     // ========== First the healthy cases =============
@@ -1636,7 +1558,7 @@ public class HBaseFsck extends Configure
         }
 
         LOG.info("Patching .META. with .regioninfo: " + hbi.getHdfsHRI());
-        HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI());
+        HBaseFsckRepair.fixMetaHoleOnline(conf, hbi.getHdfsHRI());
 
         tryAssignmentRepair(hbi, "Trying to reassign region...");
       }
@@ -1652,7 +1574,7 @@ public class HBaseFsck extends Configure
         }
 
         LOG.info("Patching .META. with with .regioninfo: " + hbi.getHdfsHRI());
-        HBaseFsckRepair.fixMetaHoleOnline(getConf(), hbi.getHdfsHRI());
+        HBaseFsckRepair.fixMetaHoleOnline(conf, hbi.getHdfsHRI());
 
         tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
       }
@@ -1814,7 +1736,7 @@ public class HBaseFsck extends Configure
     debugLsr(contained.getHdfsRegionDir());
 
     // rename the contained into the container.
-    FileSystem fs = targetRegionDir.getFileSystem(getConf());
+    FileSystem fs = targetRegionDir.getFileSystem(conf);
     FileStatus[] dirs = fs.listStatus(contained.getHdfsRegionDir());
 
     if (dirs == null) {
@@ -2429,7 +2351,7 @@ public class HBaseFsck extends Configure
     HTableDescriptor[] htd = new HTableDescriptor[0];
      try {
        LOG.info("getHTableDescriptors == tableNames => " + tableNames);
-       htd = new HBaseAdmin(getConf()).getTableDescriptors(tableNames);
+       htd = new HBaseAdmin(conf).getTableDescriptors(tableNames);
      } catch (IOException e) {
        LOG.debug("Exception getting table descriptors", e);
      }
@@ -2572,12 +2494,12 @@ public class HBaseFsck extends Configure
     };
 
     // Scan -ROOT- to pick up META regions
-    MetaScanner.metaScan(getConf(), visitor, null, null,
+    MetaScanner.metaScan(conf, visitor, null, null,
       Integer.MAX_VALUE, HConstants.ROOT_TABLE_NAME);
 
     if (!checkMetaOnly) {
       // Scan .META. to pick up user regions
-      MetaScanner.metaScan(getConf(), visitor);
+      MetaScanner.metaScan(conf, visitor);
     }
 
     errors.print("");
@@ -2831,12 +2753,6 @@ public class HBaseFsck extends Configure
     }
   }
 
-  private static ErrorReporter getErrorReporter(
-      final Configuration conf) throws ClassNotFoundException {
-    Class<? extends ErrorReporter> reporter = conf.getClass("hbasefsck.errorreporter", PrintingErrorReporter.class, ErrorReporter.class);
-    return (ErrorReporter)ReflectionUtils.newInstance(reporter, conf);
-  }
-
   public interface ErrorReporter {
     public static enum ERROR_CODE {
       UNKNOWN, NO_META_REGION, NULL_ROOT_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META,
@@ -2844,7 +2760,7 @@ public class HBaseFsck extends Configure
       MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE,
       FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
       HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
-      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE
+      ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE
     }
     public void clear();
     public void report(String message);
@@ -2862,7 +2778,7 @@ public class HBaseFsck extends Configure
     public boolean tableHasErrors(TableInfo table);
   }
 
-  static class PrintingErrorReporter implements ErrorReporter {
+  private static class PrintingErrorReporter implements ErrorReporter {
     public int errorCount = 0;
     private int showProgress;
 
@@ -3212,14 +3128,6 @@ public class HBaseFsck extends Configure
     return fixMeta;
   }
 
-  public void setCheckHdfs(boolean checking) {
-    checkHdfs = checking;
-  }
-
-  boolean shouldCheckHdfs() {
-    return checkHdfs;
-  }
-
   public void setFixHdfsHoles(boolean shouldFix) {
     fixHdfsHoles = shouldFix;
   }
@@ -3276,14 +3184,6 @@ public class HBaseFsck extends Configure
     return fixSplitParents;
   }
 
-  public void setFixReferenceFiles(boolean shouldFix) {
-    fixReferenceFiles = shouldFix;
-  }
-
-  boolean shouldFixReferenceFiles() {
-    return fixReferenceFiles;
-  }
-
   public boolean shouldIgnorePreCheckPermission() {
     return ignorePreCheckPermission;
   }
@@ -3345,7 +3245,7 @@ public class HBaseFsck extends Configure
   }
 
   protected HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
-    return new HFileCorruptionChecker(getConf(), executor, sidelineCorruptHFiles);
+    return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles);
   }
 
   public HFileCorruptionChecker getHFilecorruptionChecker() {
@@ -3383,8 +3283,6 @@ public class HBaseFsck extends Configure
     System.err.println("   -fix              Try to fix region assignments.  This is for backwards compatiblity");
     System.err.println("   -fixAssignments   Try to fix region assignments.  Replaces the old -fix");
     System.err.println("   -fixMeta          Try to fix meta problems.  This assumes HDFS region info is good.");
-    System.err.println("   -noHdfsChecking   Don't load/check region info from HDFS."
-        + " Assumes META region info is good. Won't check/fix any HDFS issue, e.g. hole, orphan, or overlap");
     System.err.println("   -fixHdfsHoles     Try to fix region holes in hdfs.");
     System.err.println("   -fixHdfsOrphans   Try to fix region dirs with no .regioninfo file in hdfs");
     System.err.println("   -fixTableOrphans  Try to fix table dirs with no .tableinfo file in hdfs (online mode only)");
@@ -3395,7 +3293,6 @@ public class HBaseFsck extends Configure
     System.err.println("   -maxOverlapsToSideline <n>  When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
     System.err.println("   -fixSplitParents  Try to force offline split parents to be online.");
     System.err.println("   -ignorePreCheckPermission  ignore filesystem permission pre-check");
-    System.err.println("   -fixReferenceFiles  Try to offline lingering reference store files");
 
     System.err.println("");
     System.err.println("  Datafile Repair options: (expert features, use with caution!)");
@@ -3405,7 +3302,7 @@ public class HBaseFsck extends Configure
     System.err.println("");
     System.err.println("  Metadata Repair shortcuts");
     System.err.println("   -repair           Shortcut for -fixAssignments -fixMeta -fixHdfsHoles " +
-        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps -fixReferenceFiles");
+        "-fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps");
     System.err.println("   -repairHoles      Shortcut for -fixAssignments -fixMeta -fixHdfsHoles");
 
     setRetCode(-2);
@@ -3419,6 +3316,7 @@ public class HBaseFsck extends Configure
    * @throws Exception
    */
   public static void main(String[] args) throws Exception {
+
     // create a fsck object
     Configuration conf = HBaseConfiguration.create();
     Path hbasedir = new Path(conf.get(HConstants.HBASE_DIR));
@@ -3426,14 +3324,12 @@ public class HBaseFsck extends Configure
     conf.set("fs.defaultFS", defaultFs.toString());     // for hadoop 0.21+
     conf.set("fs.default.name", defaultFs.toString());  // for hadoop 0.20
 
-    int ret = ToolRunner.run(new HBaseFsck(conf), args);
-    System.exit(ret);
-  }
-
-  @Override
-  public int run(String[] args) throws Exception {
-    exec(executor, args);
-    return getRetCode();
+    int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
+    ExecutorService exec = new ScheduledThreadPoolExecutor(numThreads);
+    HBaseFsck hbck = new HBaseFsck(conf, exec);
+    hbck.exec(exec, args);
+    int retcode = hbck.getRetCode();
+    Runtime.getRuntime().exit(retcode);
   }
 
   public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException,
@@ -3490,8 +3386,6 @@ public class HBaseFsck extends Configure
         setFixAssignments(true);
       } else if (cmd.equals("-fixMeta")) {
         setFixMeta(true);
-      } else if (cmd.equals("-noHdfsChecking")) {
-        setCheckHdfs(false);
       } else if (cmd.equals("-fixHdfsHoles")) {
         setFixHdfsHoles(true);
       } else if (cmd.equals("-fixHdfsOrphans")) {
@@ -3512,8 +3406,6 @@ public class HBaseFsck extends Configure
         checkCorruptHFiles = true;
       } else if (cmd.equals("-sidelineCorruptHFiles")) {
         sidelineCorruptHFiles = true;
-      } else if (cmd.equals("-fixReferenceFiles")) {
-        setFixReferenceFiles(true);
       } else if (cmd.equals("-repair")) {
         // this attempts to merge overlapping hdfs regions, needs testing
         // under load
@@ -3525,8 +3417,6 @@ public class HBaseFsck extends Configure
         setFixVersionFile(true);
         setSidelineBigOverlaps(true);
         setFixSplitParents(false);
-        setCheckHdfs(true);
-        setFixReferenceFiles(true);
       } else if (cmd.equals("-repairHoles")) {
         // this will make all missing hdfs regions available but may lose data
         setFixHdfsHoles(true);
@@ -3536,7 +3426,6 @@ public class HBaseFsck extends Configure
         setFixHdfsOverlaps(false);
         setSidelineBigOverlaps(false);
         setFixSplitParents(false);
-        setCheckHdfs(true);
       } else if (cmd.equals("-maxOverlapsToSideline")) {
         if (i == args.length - 1) {
           System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
@@ -3595,13 +3484,13 @@ public class HBaseFsck extends Configure
       setHFileCorruptionChecker(hfcc); // so we can get result
       Collection<String> tables = getIncludedTables();
       Collection<Path> tableDirs = new ArrayList<Path>();
-      Path rootdir = FSUtils.getRootDir(getConf());
+      Path rootdir = FSUtils.getRootDir(conf);
       if (tables.size() > 0) {
         for (String t : tables) {
           tableDirs.add(FSUtils.getTablePath(rootdir, t));
         }
       } else {
-        tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(getConf()), rootdir);
+        tableDirs = FSUtils.getTableDirs(FSUtils.getCurrentFileSystem(conf), rootdir);
       }
       hfcc.checkTables(tableDirs);
       PrintWriter out = new PrintWriter(System.out);
@@ -3641,14 +3530,14 @@ public class HBaseFsck extends Configure
    * ls -r for debugging purposes
    */
   void debugLsr(Path p) throws IOException {
-    debugLsr(getConf(), p);
+    debugLsr(conf, p);
   }
 
   /**
    * ls -r for debugging purposes
    */
   public static void debugLsr(Configuration conf, Path p) throws IOException {
-    if (!LOG.isDebugEnabled() || p == null) {
+    if (!LOG.isDebugEnabled()) {
       return;
     }
     FileSystem fs = p.getFileSystem(conf);

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/Threads.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/Threads.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/Threads.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/util/Threads.java Sun Dec 23 19:34:53 2012
@@ -126,7 +126,7 @@ public class Threads {
   /**
    * @param millis How long to sleep for in milliseconds.
    */
-  public static void sleep(long millis) {
+  public static void sleep(int millis) {
     try {
       Thread.sleep(millis);
     } catch (InterruptedException e) {
@@ -158,15 +158,15 @@ public class Threads {
   }
 
   /**
-   * Create a new CachedThreadPool with a bounded number as the maximum
+   * Create a new CachedThreadPool with a bounded number as the maximum 
    * thread size in the pool.
-   *
+   * 
    * @param maxCachedThread the maximum thread could be created in the pool
    * @param timeout the maximum time to wait
    * @param unit the time unit of the timeout argument
    * @param threadFactory the factory to use when creating new threads
-   * @return threadPoolExecutor the cachedThreadPool with a bounded number
-   * as the maximum thread size in the pool.
+   * @return threadPoolExecutor the cachedThreadPool with a bounded number 
+   * as the maximum thread size in the pool. 
    */
   public static ThreadPoolExecutor getBoundedCachedThreadPool(
       int maxCachedThread, long timeout, TimeUnit unit,
@@ -178,13 +178,12 @@ public class Threads {
     boundedCachedThreadPool.allowCoreThreadTimeOut(true);
     return boundedCachedThreadPool;
   }
-
-
+  
+  
   /**
-   * Returns a {@link java.util.concurrent.ThreadFactory} that names each
-   * created thread uniquely, with a common prefix.
-   *
-   * @param prefix  The prefix of every created Thread's name
+   * Returns a {@link java.util.concurrent.ThreadFactory} that names each created thread uniquely,
+   * with a common prefix.
+   * @param prefix The prefix of every created Thread's name
    * @return a {@link java.util.concurrent.ThreadFactory} that names threads
    */
   public static ThreadFactory getNamedThreadFactory(final String prefix) {
@@ -206,7 +205,6 @@ public class Threads {
     };
   }
 
-
   /**
    * Get a named {@link ThreadFactory} that just builds daemon threads
    * @param prefix name prefix for all threads created from the factory

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java Sun Dec 23 19:34:53 2012
@@ -61,12 +61,6 @@ public class HQuorumPeer {
       writeMyID(zkProperties);
       QuorumPeerConfig zkConfig = new QuorumPeerConfig();
       zkConfig.parseProperties(zkProperties);
-
-      // login the zookeeper server principal (if using security)
-      ZKUtil.loginServer(conf, "hbase.zookeeper.server.keytab.file",
-        "hbase.zookeeper.server.kerberos.principal",
-        zkConfig.getClientPortAddress().getHostName());
-
       runZKServer(zkConfig);
     } catch (Exception e) {
       e.printStackTrace();

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java Sun Dec 23 19:34:53 2012
@@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.zookeepe
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
-import java.util.LinkedList;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -33,16 +32,11 @@ import org.apache.hadoop.hbase.util.Retr
 import org.apache.zookeeper.AsyncCallback;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.Op;
-import org.apache.zookeeper.OpResult;
 import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.ZooKeeper.States;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
-import org.apache.zookeeper.proto.CreateRequest;
-import org.apache.zookeeper.proto.SetDataRequest;
 
 /**
  * A zookeeper that can handle 'recoverable' errors.
@@ -496,61 +490,6 @@ public class RecoverableZooKeeper {
     }
   }
 
-  /**
-   * Convert Iterable of {@link ZKOp} we got into the ZooKeeper.Op
-   * instances to actually pass to multi (need to do this in order to appendMetaData).
-   */
-  private Iterable<Op> prepareZKMulti(Iterable<Op> ops)
-  throws UnsupportedOperationException {
-    if(ops == null) return null;
-
-    List<Op> preparedOps = new LinkedList<Op>();
-    for (Op op : ops) {
-      if (op.getType() == ZooDefs.OpCode.create) {
-        CreateRequest create = (CreateRequest)op.toRequestRecord();
-        preparedOps.add(Op.create(create.getPath(), appendMetaData(create.getData()),
-          create.getAcl(), create.getFlags()));
-      } else if (op.getType() == ZooDefs.OpCode.delete) {
-        // no need to appendMetaData for delete
-        preparedOps.add(op);
-      } else if (op.getType() == ZooDefs.OpCode.setData) {
-        SetDataRequest setData = (SetDataRequest)op.toRequestRecord();
-        preparedOps.add(Op.setData(setData.getPath(), appendMetaData(setData.getData()),
-          setData.getVersion()));
-      } else {
-        throw new UnsupportedOperationException("Unexpected ZKOp type: " + op.getClass().getName());
-      }
-    }
-    return preparedOps;
-  }
-
-  /**
-   * Run multiple operations in a transactional manner. Retry before throwing exception
-   */
-  public List<OpResult> multi(Iterable<Op> ops)
-  throws KeeperException, InterruptedException {
-    RetryCounter retryCounter = retryCounterFactory.create();
-    Iterable<Op> multiOps = prepareZKMulti(ops);
-    while (true) {
-      try {
-        return zk.multi(multiOps);
-      } catch (KeeperException e) {
-        switch (e.code()) {
-          case CONNECTIONLOSS:
-          case SESSIONEXPIRED:
-          case OPERATIONTIMEOUT:
-            retryOrThrow(retryCounter, e, "multi");
-            break;
-
-          default:
-            throw e;
-        }
-      }
-      retryCounter.sleepUntilNextRetry();
-      retryCounter.useRetry();
-    }
-  }
-
   private String findPreviousSequentialNode(String path)
     throws KeeperException, InterruptedException {
     int lastSlashIdx = path.lastIndexOf('/');

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java Sun Dec 23 19:34:53 2012
@@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.zookeepe
 
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -30,7 +29,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
 import org.apache.zookeeper.KeeperException;
 
 /**
@@ -230,19 +228,16 @@ public class ZKTable {
       }
     }
     synchronized (this.cache) {
-      List<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
       if (settingToEnabled) {
-        ops.add(ZKUtilOp.deleteNodeFailSilent(znode92));
+        ZKUtil.deleteNodeFailSilent(this.watcher, znode92);
       }
       else {
-        ops.add(ZKUtilOp.setData(znode92, Bytes.toBytes(state.toString())));
+        ZKUtil.setData(this.watcher, znode92, Bytes.toBytes(state.toString()));
       }
-      // If not running multi-update either because of configuration or failure,
-      // set the current format znode after the 0.92 format znode.
+      // Set the current format znode after the 0.92 format znode.
       // This is so in the case of failure, the AssignmentManager is guaranteed to
       // see the state was not applied, since it uses the current format znode internally.
-      ops.add(ZKUtilOp.setData(znode, Bytes.toBytes(state.toString())));
-      ZKUtil.multiOrSequential(this.watcher, ops, true);
+      ZKUtil.setData(this.watcher, znode, Bytes.toBytes(state.toString()));
       this.cache.put(tableName, state);
     }
   }
@@ -297,16 +292,13 @@ public class ZKTable {
   public void setDeletedTable(final String tableName)
   throws KeeperException {
     synchronized (this.cache) {
-      List<ZKUtilOp> ops = new LinkedList<ZKUtilOp>();
-      ops.add(ZKUtilOp.deleteNodeFailSilent(
-        ZKUtil.joinZNode(this.watcher.masterTableZNode92, tableName)));
-      // If not running multi-update either because of configuration or failure,
-      // delete the current format znode after the 0.92 format znode.  This is so in the case of
-      // failure, the AssignmentManager is guaranteed to see the table was not deleted, since it
-      // uses the current format znode internally.
-      ops.add(ZKUtilOp.deleteNodeFailSilent(
-        ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName)));
-      ZKUtil.multiOrSequential(this.watcher, ops, true);
+      ZKUtil.deleteNodeFailSilent(this.watcher,
+        ZKUtil.joinZNode(this.watcher.masterTableZNode92, tableName));
+      // Delete the current format znode after the 0.92 format znode.
+      // This is so in the case of failure, the AssignmentManager is guaranteed to
+      // see the table was not deleted, since it uses the current format znode internally.
+      ZKUtil.deleteNodeFailSilent(this.watcher,
+        ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName));
       if (this.cache.remove(tableName) == null) {
         LOG.warn("Moving table " + tableName + " state to deleted but was " +
           "already deleted");

Modified: hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java (original)
+++ hbase/branches/0.94-test/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java Sun Dec 23 19:34:53 2012
@@ -24,21 +24,10 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.net.InetSocketAddress;
-import java.net.InetAddress;
 import java.net.Socket;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.security.auth.login.LoginException;
-import javax.security.auth.login.AppConfigurationEntry;
-import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -50,24 +39,15 @@ import org.apache.hadoop.hbase.ServerNam
 import org.apache.hadoop.hbase.executor.RegionTransitionData;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.CreateAndFailSilent;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.DeleteNodeFailSilent;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp.SetData;
 import org.apache.zookeeper.AsyncCallback;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.apache.zookeeper.Op;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooDefs.Ids;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
-import org.apache.zookeeper.client.ZooKeeperSaslClient;
-import org.apache.zookeeper.server.ZooKeeperSaslServer;
-import org.apache.zookeeper.proto.CreateRequest;
-import org.apache.zookeeper.proto.DeleteRequest;
-import org.apache.zookeeper.proto.SetDataRequest;
 
 /**
  * Internal HBase utility class for ZooKeeper.
@@ -128,170 +108,6 @@ public class ZKUtil {
         retry, retryIntervalMillis);
   }
 
-  /**
-   * Log in the current zookeeper server process using the given configuration
-   * keys for the credential file and login principal.
-   *
-   * <p><strong>This is only applicable when running on secure hbase</strong>
-   * On regular HBase (without security features), this will safely be ignored.
-   * </p>
-   *
-   * @param conf The configuration data to use
-   * @param keytabFileKey Property key used to configure the path to the credential file
-   * @param userNameKey Property key used to configure the login principal
-   * @param hostname Current hostname to use in any credentials
-   * @throws IOException underlying exception from SecurityUtil.login() call
-   */
-  public static void loginServer(Configuration conf, String keytabFileKey,
-      String userNameKey, String hostname) throws IOException {
-    login(conf, keytabFileKey, userNameKey, hostname,
-          ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
-          JaasConfiguration.SERVER_KEYTAB_KERBEROS_CONFIG_NAME);
-  }
-
-  /**
-   * Log in the current zookeeper client using the given configuration
-   * keys for the credential file and login principal.
-   *
-   * <p><strong>This is only applicable when running on secure hbase</strong>
-   * On regular HBase (without security features), this will safely be ignored.
-   * </p>
-   *
-   * @param conf The configuration data to use
-   * @param keytabFileKey Property key used to configure the path to the credential file
-   * @param userNameKey Property key used to configure the login principal
-   * @param hostname Current hostname to use in any credentials
-   * @throws IOException underlying exception from SecurityUtil.login() call
-   */
-  public static void loginClient(Configuration conf, String keytabFileKey,
-      String userNameKey, String hostname) throws IOException {
-    login(conf, keytabFileKey, userNameKey, hostname,
-          ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
-          JaasConfiguration.CLIENT_KEYTAB_KERBEROS_CONFIG_NAME);
-  }
-
-  /**
-   * Log in the current process using the given configuration keys for the
-   * credential file and login principal.
-   *
-   * <p><strong>This is only applicable when running on secure hbase</strong>
-   * On regular HBase (without security features), this will safely be ignored.
-   * </p>
-   *
-   * @param conf The configuration data to use
-   * @param keytabFileKey Property key used to configure the path to the credential file
-   * @param userNameKey Property key used to configure the login principal
-   * @param hostname Current hostname to use in any credentials
-   * @param loginContextProperty property name to expose the entry name
-   * @param loginContextName jaas entry name
-   * @throws IOException underlying exception from SecurityUtil.login() call
-   */
-  private static void login(Configuration conf, String keytabFileKey,
-      String userNameKey, String hostname,
-      String loginContextProperty, String loginContextName)
-      throws IOException {
-    if (!isSecureZooKeeper(conf))
-      return;
-
-    // User has specified a jaas.conf, keep this one as the good one.
-    // HBASE_OPTS="-Djava.security.auth.login.config=jaas.conf"
-    if (System.getProperty("java.security.auth.login.config") != null)
-      return;
-
-    String keytabFilename = conf.get(keytabFileKey);
-    String principalConfig = conf.get(userNameKey, System.getProperty("user.name"));
-    String principalName = SecurityUtil.getServerPrincipal(principalConfig, hostname);
-
-    // Initialize the "jaas.conf" for keyTab/principal,
-    // If keyTab is not specified use the Ticket Cache.
-    // and set the zookeeper login context name.
-    JaasConfiguration jaasConf = new JaasConfiguration(loginContextName,
-      keytabFilename, principalName);
-    javax.security.auth.login.Configuration.setConfiguration(jaasConf);
-    System.setProperty(loginContextProperty, loginContextName);
-  }
-
-  /**
-   * A JAAS configuration that defines the login modules that we want to use for login.
-   */
-  private static class JaasConfiguration extends javax.security.auth.login.Configuration {
-    private static final String SERVER_KEYTAB_KERBEROS_CONFIG_NAME =
-      "zookeeper-server-keytab-kerberos";
-    private static final String CLIENT_KEYTAB_KERBEROS_CONFIG_NAME =
-      "zookeeper-client-keytab-kerberos";
-
-    private static final Map<String, String> BASIC_JAAS_OPTIONS =
-      new HashMap<String,String>();
-    static {
-      String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG");
-      if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
-        BASIC_JAAS_OPTIONS.put("debug", "true");
-      }
-    }
-
-    private static final Map<String,String> KEYTAB_KERBEROS_OPTIONS =
-      new HashMap<String,String>();
-    static {
-      KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true");
-      KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true");
-      KEYTAB_KERBEROS_OPTIONS.put("refreshKrb5Config", "true");
-      KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS);
-    }
-
-    private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN =
-      new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
-                                LoginModuleControlFlag.REQUIRED,
-                                KEYTAB_KERBEROS_OPTIONS);
-
-    private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF =
-      new AppConfigurationEntry[]{KEYTAB_KERBEROS_LOGIN};
-
-    private javax.security.auth.login.Configuration baseConfig;
-    private final String loginContextName;
-    private final boolean useTicketCache;
-    private final String keytabFile;
-    private final String principal;
-
-    public JaasConfiguration(String loginContextName, String principal) {
-      this(loginContextName, principal, null, true);
-    }
-
-    public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
-      this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
-    }
-
-    private JaasConfiguration(String loginContextName, String principal,
-                             String keytabFile, boolean useTicketCache) {
-      try {
-        this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
-      } catch (SecurityException e) {
-        this.baseConfig = null;
-      }
-      this.loginContextName = loginContextName;
-      this.useTicketCache = useTicketCache;
-      this.keytabFile = keytabFile;
-      this.principal = principal;
-      LOG.info("JaasConfiguration loginContextName=" + loginContextName +
-               " principal=" + principal + " useTicketCache=" + useTicketCache +
-               " keytabFile=" + keytabFile);
-    }
-
-    @Override
-    public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
-      if (loginContextName.equals(appName)) {
-        if (!useTicketCache) {
-          KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile);
-          KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true");
-        }
-        KEYTAB_KERBEROS_OPTIONS.put("principal", principal);
-        KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", useTicketCache ? "true" : "false");
-        return KEYTAB_KERBEROS_CONF;
-      }
-      if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName);
-      return(null);
-    }
-  }
-
   //
   // Helper methods
   //
@@ -433,6 +249,9 @@ public class ZKUtil {
   /**
    * Check if the specified node exists.  Sets no watches.
    *
+   * Returns true if node exists, false if not.  Returns an exception if there
+   * is an unexpected zookeeper exception.
+   *
    * @param zkw zk reference
    * @param znode path of node to watch
    * @return version of the node if it exists, -1 if does not exist
@@ -881,29 +700,19 @@ public class ZKUtil {
    */
   public static void setData(ZooKeeperWatcher zkw, String znode, byte [] data)
   throws KeeperException, KeeperException.NoNodeException {
-    setData(zkw, (SetData)ZKUtilOp.setData(znode, data));
+    setData(zkw, znode, data, -1);
   }
 
-  private static void setData(ZooKeeperWatcher zkw, SetData setData)
-  throws KeeperException, KeeperException.NoNodeException {
-    SetDataRequest sd = (SetDataRequest)toZooKeeperOp(zkw, setData).toRequestRecord();
-    setData(zkw, sd.getPath(), sd.getData(), sd.getVersion());
-  }
-
-  /**
-   * Returns whether or not secure authentication is enabled
-   * (whether <code>hbase.security.authentication</code> is set to
-   * <code>kerberos</code>.
-   */
   public static boolean isSecureZooKeeper(Configuration conf) {
-    // hbase shell need to use:
-    //    -Djava.security.auth.login.config=user-jaas.conf
-    // since each user has a different jaas.conf
-    if (System.getProperty("java.security.auth.login.config") != null)
-      return true;
-
-    // Master & RSs uses hbase.zookeeper.client.*
-    return "kerberos".equalsIgnoreCase(conf.get("hbase.security.authentication"));
+    // TODO: We need a better check for security enabled ZooKeeper. Currently
+    // the secure ZooKeeper client is set up using a supplied JaaS
+    // configuration file. But if the system property for the JaaS
+    // configuration file is set, this may not be an exclusive indication
+    // that HBase should set ACLs on znodes. As an alternative, we could do
+    // this more like Hadoop and build a JaaS configuration programmatically
+    // based on a site conf setting. The scope of such a change will be
+    // addressed in HBASE-4791.
+    return (System.getProperty("java.security.auth.login.config") != null);
   }
 
   private static ArrayList<ACL> createACL(ZooKeeperWatcher zkw, String node) {
@@ -1087,20 +896,14 @@ public class ZKUtil {
    * @throws KeeperException if unexpected zookeeper exception
    */
   public static void createAndFailSilent(ZooKeeperWatcher zkw,
-      String znode) throws KeeperException {
-    createAndFailSilent(zkw,
-      (CreateAndFailSilent)ZKUtilOp.createAndFailSilent(znode, new byte[0]));
-  }
-
-  private static void createAndFailSilent(ZooKeeperWatcher zkw, CreateAndFailSilent cafs)
+      String znode)
   throws KeeperException {
-    CreateRequest create = (CreateRequest)toZooKeeperOp(zkw, cafs).toRequestRecord();
-    String znode = create.getPath();
     try {
       RecoverableZooKeeper zk = zkw.getRecoverableZooKeeper();
       waitForZKConnectionIfAuthenticating(zkw);
       if (zk.exists(znode, false) == null) {
-        zk.create(znode, create.getData(), create.getAcl(), CreateMode.fromFlag(create.getFlags()));
+        zk.create(znode, new byte[0], createACL(zkw,znode),
+            CreateMode.PERSISTENT);
       }
     } catch(KeeperException.NodeExistsException nee) {
     } catch(KeeperException.NoAuthException nee){
@@ -1186,15 +989,8 @@ public class ZKUtil {
    */
   public static void deleteNodeFailSilent(ZooKeeperWatcher zkw, String node)
   throws KeeperException {
-    deleteNodeFailSilent(zkw,
-      (DeleteNodeFailSilent)ZKUtilOp.deleteNodeFailSilent(node));
-  }
-
-  private static void deleteNodeFailSilent(ZooKeeperWatcher zkw,
-      DeleteNodeFailSilent dnfs) throws KeeperException {
-    DeleteRequest delete = (DeleteRequest)toZooKeeperOp(zkw, dnfs).toRequestRecord();
     try {
-      zkw.getRecoverableZooKeeper().delete(delete.getPath(), delete.getVersion());
+      zkw.getRecoverableZooKeeper().delete(node, -1);
     } catch(KeeperException.NoNodeException nne) {
     } catch(InterruptedException ie) {
       zkw.interruptedException(ie);
@@ -1242,209 +1038,6 @@ public class ZKUtil {
     }
   }
 
-  /**
-   * Represents an action taken by ZKUtil, e.g. createAndFailSilent.
-   * These actions are higher-level than {@link ZKOp} actions, which represent
-   * individual actions in the ZooKeeper API, like create.
-   */
-  public abstract static class ZKUtilOp {
-    private String path;
-
-    private ZKUtilOp(String path) {
-      this.path = path;
-    }
-
-    /**
-     * @return a createAndFailSilent ZKUtilOp
-     */
-    public static ZKUtilOp createAndFailSilent(String path, byte[] data) {
-      return new CreateAndFailSilent(path, data);
-    }
-
-    /**
-     * @return a deleteNodeFailSilent ZKUtilOP
-     */
-    public static ZKUtilOp deleteNodeFailSilent(String path) {
-      return new DeleteNodeFailSilent(path);
-    }
-
-    /**
-     * @return a setData ZKUtilOp
-     */
-    public static ZKUtilOp setData(String path, byte [] data) {
-      return new SetData(path, data);
-    }
-
-    /**
-     * @return path to znode where the ZKOp will occur
-     */
-    public String getPath() {
-      return path;
-    }
-
-    /**
-     * ZKUtilOp representing createAndFailSilent in ZooKeeper
-     * (attempt to create node, ignore error if already exists)
-     */
-    public static class CreateAndFailSilent extends ZKUtilOp {
-      private byte [] data;
-
-      private CreateAndFailSilent(String path, byte [] data) {
-        super(path);
-        this.data = data;
-      }
-
-      public byte[] getData() {
-        return data;
-      }
-
-      @Override
-      public boolean equals(Object o) {
-        if (this == o) return true;
-        if (!(o instanceof CreateAndFailSilent)) return false;
-
-        CreateAndFailSilent op = (CreateAndFailSilent) o;
-        return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
-      }
-    }
-
-    /**
-     * ZKUtilOp representing deleteNodeFailSilent in ZooKeeper
-     * (attempt to delete node, ignore error if node doesn't exist)
-     */
-    public static class DeleteNodeFailSilent extends ZKUtilOp {
-      private DeleteNodeFailSilent(String path) {
-        super(path);
-      }
-
-      @Override
-      public boolean equals(Object o) {
-        if (this == o) return true;
-        if (!(o instanceof DeleteNodeFailSilent)) return false;
-
-        return super.equals(o);
-      }
-    }
-
-    /**
-     * @return ZKUtilOp representing setData in ZooKeeper
-     */
-    public static class SetData extends ZKUtilOp {
-      private byte [] data;
-
-      private SetData(String path, byte [] data) {
-        super(path);
-        this.data = data;
-      }
-
-      public byte[] getData() {
-        return data;
-      }
-
-      @Override
-      public boolean equals(Object o) {
-        if (this == o) return true;
-        if (!(o instanceof SetData)) return false;
-
-        SetData op = (SetData) o;
-        return getPath().equals(op.getPath()) && Arrays.equals(data, op.data);
-      }
-    }
-  }
-
-  /**
-   * Convert from ZKUtilOp to ZKOp
-   */
-  private static Op toZooKeeperOp(ZooKeeperWatcher zkw, ZKUtilOp op)
-  throws UnsupportedOperationException {
-    if(op == null) return null;
-
-    if (op instanceof CreateAndFailSilent) {
-      CreateAndFailSilent cafs = (CreateAndFailSilent)op;
-      return Op.create(cafs.getPath(), cafs.getData(), createACL(zkw, cafs.getPath()),
-        CreateMode.PERSISTENT);
-    } else if (op instanceof DeleteNodeFailSilent) {
-      DeleteNodeFailSilent dnfs = (DeleteNodeFailSilent)op;
-      return Op.delete(dnfs.getPath(), -1);
-    } else if (op instanceof SetData) {
-      SetData sd = (SetData)op;
-      return Op.setData(sd.getPath(), sd.getData(), -1);
-    } else {
-      throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
-        + op.getClass().getName());
-    }
-  }
-
-  /**
-   * If hbase.zookeeper.useMulti is true, use ZooKeeper's multi-update functionality.
-   * Otherwise, run the list of operations sequentially.
-   *
-   * If all of the following are true:
-   * - runSequentialOnMultiFailure is true
-   * - hbase.zookeeper.useMulti is true
-   * - on calling multi, we get a ZooKeeper exception that can be handled by a sequential call(*)
-   * Then:
-   * - we retry the operations one-by-one (sequentially)
-   *
-   * Note *: an example is receiving a NodeExistsException from a "create" call.  Without multi,
-   * a user could call "createAndFailSilent" to ensure that a node exists if they don't care who
-   * actually created the node (i.e. the NodeExistsException from ZooKeeper is caught).
-   * This will cause all operations in the multi to fail, however, because
-   * the NodeExistsException that zk.create throws will fail the multi transaction.
-   * In this case, if the previous conditions hold, the commands are run sequentially, which should
-   * result in the correct final state, but means that the operations will not run atomically.
-   *
-   * @throws KeeperException
-   */
-  public static void multiOrSequential(ZooKeeperWatcher zkw, List<ZKUtilOp> ops,
-      boolean runSequentialOnMultiFailure) throws KeeperException {
-    if (ops == null) return;
-    boolean useMulti = zkw.getConfiguration().getBoolean(HConstants.ZOOKEEPER_USEMULTI, false);
-
-    if (useMulti) {
-      List<Op> zkOps = new LinkedList<Op>();
-      for (ZKUtilOp op : ops) {
-        zkOps.add(toZooKeeperOp(zkw, op));
-      }
-      try {
-        zkw.getRecoverableZooKeeper().multi(zkOps);
-      } catch (KeeperException ke) {
-       switch (ke.code()) {
-         case NODEEXISTS:
-         case NONODE:
-         case BADVERSION:
-         case NOAUTH:
-           // if we get an exception that could be solved by running sequentially
-           // (and the client asked us to), then break out and run sequentially
-           if (runSequentialOnMultiFailure) {
-             LOG.info("On call to ZK.multi, received exception: " + ke.toString() + "."
-               + "  Attempting to run operations sequentially because"
-               + " runSequentialOnMultiFailure is: " + runSequentialOnMultiFailure + ".");
-             break;
-           }
-          default:
-            throw ke;
-        }
-      } catch (InterruptedException ie) {
-        zkw.interruptedException(ie);
-      }
-    }
-
-    // run sequentially
-    for (ZKUtilOp op : ops) {
-      if (op instanceof CreateAndFailSilent) {
-        createAndFailSilent(zkw, (CreateAndFailSilent)op);
-      } else if (op instanceof DeleteNodeFailSilent) {
-        deleteNodeFailSilent(zkw, (DeleteNodeFailSilent)op);
-      } else if (op instanceof SetData) {
-        setData(zkw, (SetData)op);
-      } else {
-        throw new UnsupportedOperationException("Unexpected ZKUtilOp type: "
-          + op.getClass().getName());
-      }
-    }
-  }
-
   //
   // ZooKeeper cluster information
   //

Modified: hbase/branches/0.94-test/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.94-test/src/main/resources/hbase-default.xml?rev=1425513&r1=1425512&r2=1425513&view=diff
==============================================================================
--- hbase/branches/0.94-test/src/main/resources/hbase-default.xml (original)
+++ hbase/branches/0.94-test/src/main/resources/hbase-default.xml Sun Dec 23 19:34:53 2012
@@ -690,17 +690,6 @@
     for more information.
     </description>
   </property>
-  <property>
-    <name>hbase.zookeeper.useMulti</name>
-    <value>false</value>
-    <description>Instructs HBase to make use of ZooKeeper's multi-update functionality.
-    This allows certain ZooKeeper operations to complete more quickly and prevents some issues
-    with rare ZooKeeper failure scenarios (see the release note of HBASE-6710 for an example).
-    IMPORTANT: only set this to true if all ZooKeeper servers in the cluster are on version 3.4+
-    and will not be downgraded.  ZooKeeper versions before 3.4 do not support multi-update and will
-    not fail gracefully if multi-update is invoked (see ZOOKEEPER-1495).
-    </description>
-  </property>
   <!-- End of properties used to generate ZooKeeper host:port quorum list. -->
 
   <!--



Mime
View raw message