hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r794530 - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./ src/java/ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/regionserver/ src/java/org/apache/hadoop/hbase/util/ src/test/data/ src/test/org/apache/hadoop/hbase/regionse...
Date Thu, 16 Jul 2009 05:09:54 GMT
Author: apurtell
Date: Thu Jul 16 05:09:53 2009
New Revision: 794530

URL: http://svn.apache.org/viewvc?rev=794530&view=rev
Log:
HBASE-1658, HBASE-1659, HBASE-1603, HBASE-1632, HBASE-1662

Added:
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/hbase-0.19-two-small-tables.zip   (with
props)
Removed:
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/testdata/
Modified:
    hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp
    hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Thu Jul 16 05:09:53 2009
@@ -265,6 +265,8 @@
    HBASE-1650  HBASE-1551 broke the ability to manage non-regionserver
                start-up/shut down. ie: you cant start/stop thrift on a cluster
                anymore
+   HBASE-1658  Remove UI refresh -- its annoying
+   HBASE-1659  merge tool doesnt take binary regions with \x escape format
 
   IMPROVEMENTS
    HBASE-1089  Add count of regions on filesystem to master UI; add percentage
@@ -474,9 +476,12 @@
    HBASE-1640  Allow passing arguments to jruby script run when run by bin/hbase shell
    HBASE-698   HLog recovery is not performed after master failure
    HBASE-1643  ScanDeleteTracker takes comparator but it unused
-   HBASE-1603  MR failed "RetriesExhaustedException: Trying to contact region server
-               Some server for region TestTable..." -- deubugging
+   HBASE-1603  MR failed "RetriesExhaustedException: Trying to contact region
+               server Some server for region TestTable..." -- deubugging
    HBASE-1470  hbase and HADOOP-4379, dhruba's flush/sync
+   HBASE-1632  Write documentation for configuring/managing ZooKeeper with HBase
+   HBASE-1662  Tool to run major compaction on catalog regions when hbase is
+               shutdown
 
   OPTIMIZATIONS
    HBASE-1412  Change values for delete column and column family in KeyValue

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HConstants.java Thu
Jul 16 05:09:53 2009
@@ -47,8 +47,9 @@
    * Version 4 supports only one kind of bloom filter.
    * Version 5 changes versions in catalog table regions.
    * Version 6 enables blockcaching on catalog tables.
+   * Version 7 introduces hfile -- hbase 0.19 to 0.20..
    */
-  public static final String FILE_SYSTEM_VERSION = "6";
+  public static final String FILE_SYSTEM_VERSION = "7";
   
   // Configuration parameters
   

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Thu Jul 16 05:09:53 2009
@@ -56,6 +56,7 @@
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.Reference.Range;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -1674,14 +1675,14 @@
     private byte [] stopRow;
     
     RegionScanner(Scan scan) {
-      if(Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
+      if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
         this.stopRow = null;
       } else {
         this.stopRow = scan.getStopRow();
       }
       
       List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
-      for(Map.Entry<byte[], NavigableSet<byte[]>> entry : 
+      for (Map.Entry<byte[], NavigableSet<byte[]>> entry : 
           scan.getFamilyMap().entrySet()) {
         Store store = stores.get(entry.getKey());
         scanners.add(store.getScanner(scan, entry.getValue()));
@@ -1703,21 +1704,20 @@
     throws IOException {
       // This method should probably be reorganized a bit... has gotten messy
       KeyValue kv = this.storeHeap.peek();
-      if(kv == null) {
+      if (kv == null) {
         return false;
       }
       byte [] currentRow = kv.getRow();
       // See if we passed stopRow
-      if(stopRow != null &&
-          comparator.compareRows(stopRow, 0, stopRow.length, 
-              currentRow, 0, currentRow.length)
-          <= 0) {
+      if (stopRow != null &&
+        comparator.compareRows(stopRow, 0, stopRow.length,
+          currentRow, 0, currentRow.length) <= 0) {
         return false;
       }
       this.storeHeap.next(results);
       while(true) {
         kv = this.storeHeap.peek();
-        if(kv == null) {
+        if (kv == null) {
           return false;
         }
         byte [] row = kv.getRow();
@@ -2348,4 +2348,106 @@
     }
     return heapSize;
   }
-}
+
+  /*
+   * This method calls System.exit.
+   * @param message Message to print out.  May be null.
+   */
+  private static void printUsageAndExit(final String message) {
+    if (message != null && message.length() > 0) System.out.println(message);
+    System.out.println("Usage: HRegion CATLALOG_TABLE_DIR [major_compact]");
+    System.out.println("Options:");
+    System.out.println(" major_compact  Pass this option to major compact " +
+      "passed region.");
+    System.out.println("Default outputs scan of passed region.");
+    System.exit(1);
+  }
+
+  /*
+   * Process table.
+   * Do major compaction or list content.
+   * @param fs
+   * @param p
+   * @param log
+   * @param c
+   * @param majorCompact
+   * @throws IOException
+   */
+  private static void processTable(final FileSystem fs, final Path p,
+      final HLog log, final HBaseConfiguration c,
+      final boolean majorCompact)
+  throws IOException {
+    HRegion region = null;
+    String rootStr = Bytes.toString(HConstants.ROOT_TABLE_NAME);
+    String metaStr = Bytes.toString(HConstants.META_TABLE_NAME);
+    // Currently expects tables have one region only.
+    if (p.getName().startsWith(rootStr)) {
+      region = new HRegion(p, log, fs, c, HRegionInfo.ROOT_REGIONINFO, null);
+    } else if (p.getName().startsWith(metaStr)) {
+      region = new HRegion(p, log, fs, c, HRegionInfo.FIRST_META_REGIONINFO,
+          null);
+    } else {
+      throw new IOException("Not a known catalog table: " + p.toString());
+    }
+    try {
+      region.initialize(null, null);
+      if (majorCompact) {
+        region.compactStores(true);
+      } else {
+        // Default behavior
+        Scan scan = new Scan();
+        InternalScanner scanner = region.getScanner(scan);
+        try {
+          List<KeyValue> kvs = new ArrayList<KeyValue>();
+          boolean done = false;
+          do {
+            kvs.clear();
+            done = scanner.next(kvs);
+            if (kvs.size() > 0) LOG.info(kvs);
+          } while (done);
+        } finally {
+          scanner.close();
+        }
+      }
+    } finally {
+      region.close();
+    }
+  }
+
+  /**
+   * Facility for dumping and compacting catalog tables.
+   * Only does catalog tables since these are only tables we for sure know
+   * schema on.  For usage run:
+   * <pre>
+   *   ./bin/hbase org.apache.hadoop.hbase.regionserver.HRegion
+   * </pre>
+   * @param args
+   * @throws IOException 
+   */
+  public static void main(String[] args) throws IOException {
+    if (args.length < 1) {
+      printUsageAndExit(null);
+    }
+    boolean majorCompact = false;
+    if (args.length > 1) {
+      if (!args[1].toLowerCase().startsWith("major")) {
+        printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">");
+      }
+      majorCompact = true;
+    
+    }
+    Path tableDir  = new Path(args[0]);
+    HBaseConfiguration c = new HBaseConfiguration();
+    FileSystem fs = FileSystem.get(c);
+    Path logdir = new Path(c.get("hbase.tmp.dir"),
+      "hlog" + tableDir.getName() + System.currentTimeMillis());
+    HLog log = new HLog(fs, logdir, c, null);
+    try {
+      processTable(fs, tableDir, log, c, majorCompact);
+     } finally {
+       log.close();
+       BlockCache bc = StoreFile.getBlockCache(c);
+       if (bc != null) bc.shutdown();
+     }
+  }
+}
\ No newline at end of file

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java
Thu Jul 16 05:09:53 2009
@@ -30,8 +30,10 @@
 import org.apache.hadoop.dfs.DistributedFileSystem;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -265,4 +267,128 @@
       HRegion.getRegionDir(rootdir, HRegionInfo.ROOT_REGIONINFO);
     return fs.exists(rootRegionDir);
   }
+
+  /**
+   * Runs through the hbase rootdir and checks all stores have only
+   * one file in them -- that is, they've been major compacted.  Looks
+   * at root and meta tables too.
+   * @param fs
+   * @param hbaseRootDir
+   * @return True if this hbase install is major compacted.
+   * @throws IOException
+   */
+  public static boolean isMajorCompacted(final FileSystem fs,
+      final Path hbaseRootDir)
+  throws IOException {
+    // Presumes any directory under hbase.rootdir is a table.
+    FileStatus [] directories = fs.listStatus(hbaseRootDir, new DirFilter(fs));
+    for (int i = 0; i < directories.length; i++) {
+      // Skip the .log directory.  All others should be tables.  Inside a table,
+      // there are compaction.dir directories to skip.  Otherwise, all else
+      // should be regions.  Then in each region, should only be family
+      // directories.  Under each of these, should be one file only.
+      Path d = directories[i].getPath();
+      if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue;
+      FileStatus [] tablesubdirectories = fs.listStatus(d, new DirFilter(fs));
+      for (int j = 0; j < tablesubdirectories.length; j++) {
+        Path dd = tablesubdirectories[j].getPath();
+        if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue;
+        // Else its a region name.  Now look in region for families.
+        FileStatus [] familydirectories = fs.listStatus(dd, new DirFilter(fs));
+        for (int k = 0; k < familydirectories.length; k++) {
+          Path family = familydirectories[k].getPath();
+          // Now in family make sure only one file.
+          FileStatus [] familyStatus = fs.listStatus(family);
+          if (familyStatus.length > 1) {
+            LOG.debug(family.toString() + " has " + familyStatus.length +
+              " files.");
+            return false;
+          }
+        }
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Expects to find -ROOT- directory.
+   * @param fs
+   * @param hbaseRootDir
+   * @return True if this a pre020 layout.
+   * @throws IOException
+   */
+  public static boolean isPre020FileLayout(final FileSystem fs,
+    final Path hbaseRootDir)
+  throws IOException {
+    Path mapfiles = new Path(new Path(new Path(hbaseRootDir, "-ROOT-"),
+      "70236052"), "mapfiles");
+    return fs.exists(mapfiles);
+  }
+
+  /**
+   * Runs through the hbase rootdir and checks all stores have only
+   * one file in them -- that is, they've been major compacted.  Looks
+   * at root and meta tables too.  This version differs from
+   * {@link #isMajorCompacted(FileSystem, Path)} in that it expects a
+   * pre-0.20.0 hbase layout on the filesystem.  Used migrating.
+   * @param fs
+   * @param hbaseRootDir
+   * @return True if this hbase install is major compacted.
+   * @throws IOException
+   */
+  public static boolean isMajorCompactedPre020(final FileSystem fs,
+      final Path hbaseRootDir)
+  throws IOException {
+    // Presumes any directory under hbase.rootdir is a table.
+    FileStatus [] directories = fs.listStatus(hbaseRootDir, new DirFilter(fs));
+    for (int i = 0; i < directories.length; i++) {
+      // Inside a table, there are compaction.dir directories to skip.
+      // Otherwise, all else should be regions.  Then in each region, should
+      // only be family directories.  Under each of these, should be a mapfile
+      // and info directory and in these only one file.
+      Path d = directories[i].getPath();
+      if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue;
+      FileStatus [] tablesubdirectories = fs.listStatus(d, new DirFilter(fs));
+      for (int j = 0; j < tablesubdirectories.length; j++) {
+        Path dd = tablesubdirectories[j].getPath();
+        if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue;
+        // Else its a region name.  Now look in region for families.
+        FileStatus [] familydirectories = fs.listStatus(dd, new DirFilter(fs));
+        for (int k = 0; k < familydirectories.length; k++) {
+          Path family = familydirectories[k].getPath();
+          // Now in family, there are 'mapfile' and 'info' subdirs.  Just
+          // look in the 'mapfile' subdir.
+          FileStatus [] familyStatus =
+            fs.listStatus(new Path(family, "mapfiles"));
+          if (familyStatus.length > 1) {
+            LOG.debug(family.toString() + " has " + familyStatus.length +
+              " files.");
+            return false;
+          }
+        }
+      }
+    }
+    return true;
+  }
+
+  /**
+   * A {@link PathFilter} that returns directories.
+   */
+  public static class DirFilter implements PathFilter {
+    private final FileSystem fs;
+
+    public DirFilter(final FileSystem fs) {
+      this.fs = fs;
+    }
+
+    public boolean accept(Path p) {
+      boolean isdir = false;
+      try {
+        isdir = this.fs.getFileStatus(p).isDir();
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+      return isdir;
+    }
+  }
 }
\ No newline at end of file

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Merge.java Thu
Jul 16 05:09:53 2009
@@ -345,8 +345,8 @@
     tableName = Bytes.toBytes(remainingArgs[0]);
     isMetaTable = Bytes.compareTo(tableName, HConstants.META_TABLE_NAME) == 0;
     
-    region1 = Bytes.toBytes(remainingArgs[1]);
-    region2 = Bytes.toBytes(remainingArgs[2]);
+    region1 = Bytes.toBytesBinary(remainingArgs[1]);
+    region2 = Bytes.toBytesBinary(remainingArgs[2]);
     int status = 0;
     if (notInTable(tableName, region1) || notInTable(tableName, region2)) {
       status = -1;

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
Thu Jul 16 05:09:53 2009
@@ -35,7 +35,6 @@
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Tool;
@@ -86,7 +85,7 @@
   private static final float HBASE_0_1_VERSION = 0.1f;
   
   // Filesystem version we can migrate from
-  private static final int PREVIOUS_VERSION = 4;
+  private static final int PREVIOUS_VERSION = 6;
   
   private static final String MIGRATION_LINK = 
     " See http://wiki.apache.org/hadoop/Hbase/HowToMigrate for more information.";
@@ -185,14 +184,14 @@
       if (version == HBASE_0_1_VERSION ||
           Integer.valueOf(versionStr).intValue() < PREVIOUS_VERSION) {
         String msg = "Cannot upgrade from " + versionStr + " to " +
-        HConstants.FILE_SYSTEM_VERSION + " you must install hbase-0.2.x, run " +
+        HConstants.FILE_SYSTEM_VERSION + " you must install an earlier hbase, run " +
         "the upgrade tool, reinstall this version and run this utility again." +
         MIGRATION_LINK;
         System.out.println(msg);
         throw new IOException(msg);
       }
 
-      migrate4To6();
+      migrate6to7();
 
       if (!readOnly) {
         // Set file system version
@@ -209,17 +208,34 @@
     }
   }
   
-  // Move the fileystem version from 4 to 6.
-  // In here we rewrite the catalog table regions so they keep 10 versions
-  // instead of 1.
-  private void migrate4To6() throws IOException {
+  // Move the fileystem version from 6 to 7.
+  private void migrate6to7() throws IOException {
     if (this.readOnly && this.migrationNeeded) {
       return;
     }
+    // Before we start, make sure all is major compacted.
+    Path hbaseRootDir = new Path(conf.get(HConstants.HBASE_DIR));
+    boolean pre020 = FSUtils.isPre020FileLayout(fs, hbaseRootDir);
+    if (pre020) {
+      if (!FSUtils.isMajorCompactedPre020(fs, hbaseRootDir)) {
+        String msg = "All tables must be major compacted before migration." +
+          MIGRATION_LINK;
+        System.out.println(msg);
+        throw new IOException(msg);
+      }
+      // TODO: Rewrite regions.
+    }
+    // TOOD: Verify all has been brought over from old to new layout.
     final MetaUtils utils = new MetaUtils(this.conf);
     try {
-      // These two operations are effectively useless.  -ROOT- is hardcode,
-      // at least until hbase 0.20.0 when we store it out in ZK.
+      // TODO: Set the .META. and -ROOT- to flush at 16k?  32k?
+      // TODO: Enable block cache on all tables
+      // TODO: Rewrite MEMCACHE_FLUSHSIZE as MEMSTORE_FLUSHSIZE – name has changed.

+      // TODO: Remove tableindexer 'index' attribute index from TableDescriptor (See HBASE-1586)

+      // TODO: TODO: Move of in-memory parameter from table to column family (from HTD to
HCD). 
+      // TODO: Purge isInMemory, etc., methods from HTD as part of migration. 
+      // TODO: Clean up old region log files (HBASE-698) 
+      
       updateVersions(utils.getRootRegion().getRegionInfo());
       enableBlockCache(utils.getRootRegion().getRegionInfo());
       // Scan the root region
@@ -235,6 +251,20 @@
           return true;
         }
       });
+      LOG.info("TODO: Note on make sure not using old hbase-default.xml");
+      /*
+       * hbase.master / hbase.master.hostname are obsolete, that's replaced by
+hbase.cluster.distributed. This config must be set to "true" to have a
+fully-distributed cluster and the server lines in zoo.cfg must not
+point to "localhost".
+
+The clients must have a valid zoo.cfg in their classpath since we
+don't provide the master address.
+
+hbase.master.dns.interface and hbase.master.dns.nameserver should be
+set to control the master's address (not mandatory).
+       */
+      LOG.info("TODO: Note on zookeeper config. before starting:");
     } finally {
       utils.shutdown();
     }

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html Thu Jul 16 05:09:53 2009
@@ -174,51 +174,99 @@
 </p>
 <p>
 A distributed HBase depends on a running ZooKeeper cluster.
-The ZooKeeper configuration file for HBase is stored at <code>${HBASE_HOME}/conf/zoo.cfg</code>.
-See the ZooKeeper <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html">
Getting Started Guide</a>
-for information about the format and options of that file.  Specifically, look at the 
-<a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html#sc_RunningReplicatedZooKeeper">Running
Replicated ZooKeeper</a> section.
-</p>
-
-
-<p>
-Though not recommended, it can be convenient having HBase continue to manage
-ZooKeeper even when in distributed mode (It can be good when testing or taking
-hbase for a testdrive).  Change <code>${HBASE_HOME}/conf/zoo.cfg</code> and
-set the server.0 property to the IP of the node that will be running ZooKeeper
-(Leaving the default value of "localhost" will make it impossible to start HBase).
-<pre>
-  ...
-server.0=example.org:2888:3888
-<blockquote>
-</pre>
-Then on the example.org server do the following <i>before</i> running HBase.

-<pre>
-${HBASE_HOME}/bin/hbase-daemon.sh start zookeeper
-</pre>
-</blockquote>
-<p>To stop ZooKeeper, after you've shut down hbase, do:
-<blockquote>
-<pre>
-${HBASE_HOME}/bin/hbase-daemon.sh stop zookeeper
-</pre>
-</blockquote>
-Be aware that this option is only recommanded for testing purposes as a failure
-on that node would render HBase <b>unusable</b>.
-</p>
-
-<p>
-To tell HBase to stop managing a ZooKeeper instance, after configuring
-<code>zoo.cfg</code> to point at the ZooKeeper Quorum you'd like HBase to
-use, in <code>${HBASE_HOME}/conf/hbase-env.sh</code>,
-set the following to tell HBase to STOP managing its instance of ZooKeeper.
-<blockquote>
-<pre>
-  ...
-# Tell HBase whether it should manage it's own instance of Zookeeper or not.
-export HBASE_MANAGES_ZK=false
-</pre>
-</blockquote>
+HBase can manage a ZooKeeper cluster for you, or you can manage it on your own
+and point HBase to it.
+To toggle this option, use the <code>HBASE_MANAGES_ZK</code> variable in <code>
+${HBASE_HOME}/conf/hbase-env.sh</code>.
+This variable, which defaults to <code>true</code>, tells HBase whether to
+start/stop the ZooKeeper quorum servers alongside the rest of the servers.
+</p>
+<p>
+To point HBase at an existing ZooKeeper cluster, add your <code>zoo.cfg</code>
+to the <code>CLASSPATH</code>.
+HBase will see this file and use it to figure out where ZooKeeper is.
+Additionally set <code>HBASE_MANAGES_ZK</code> in <code> ${HBASE_HOME}/conf/hbase-env.sh</code>
+ to <code>false</code> so that HBase doesn't mess with your ZooKeeper setup:
+<pre>
+   ...
+  # Tell HBase whether it should manage it's own instance of Zookeeper or not.
+  export HBASE_MANAGES_ZK=false
+</pre>
+For more information about setting up a ZooKeeper cluster on your own, see
+the ZooKeeper <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html">Getting
Started Guide</a>.
+HBase currently uses ZooKeeper version 3.2.0, so any cluster setup with a 3.x.x
+version of ZooKeeper should work.
+</p>
+<p>
+To have HBase manage the ZooKeeper cluster, you can use a <code>zoo.cfg</code>
+ file as above, or edit the options directly in the <code>${HBASE_HOME}/conf/hbase-site.xml</code>.
+Every option from the <code>zoo.cfg</code> has a corresponding property in the
+XML configuration file named <code>hbase.zookeeper.property.OPTION</code>.
+For example, the <code>clientPort</code> setting in ZooKeeper can be changed
by
+setting the <code>hbase.zookeeper.property.clientPort</code> property.
+For the full list of available properties, see ZooKeeper's <code>zoo.cfg</code>.
+For the default values used by HBase, see <code>${HBASE_HOME}/conf/hbase-default.xml</code>.
+</p>
+<p>
+At minimum, you should set the list of servers that you want ZooKeeper to run
+on using the <code>hbase.zookeeper.quorum</code> property.
+This property defaults to <code>localhost</code> which is not suitable for a

+fully distributed HBase.
+It is recommended to run a ZooKeeper quorum of 5 or 7 machines, and give each
+server around 1GB to ensure that they don't swap.
+It is also recommended to run the ZooKeeper servers on separate machines from
+the Region Servers with their own disks.
+If this is not easily doable for you, choose 5 of your region servers to run the
+ZooKeeper servers on.
+</p>
+<p>
+As an example, to have HBase manage a ZooKeeper quorum on nodes
+rs{1,2,3,4,5}.example.com, bound to port 2222 (the default is 2181), use:
+<pre>
+  ${HBASE_HOME}/conf/hbase-env.sh:
+
+       ...
+      # Tell HBase whether it should manage it's own instance of Zookeeper or not.
+      export HBASE_MANAGES_ZK=true
+
+  ${HBASE_HOME}/conf/hbase-site.xml:
+
+  &lt;configuration&gt;
+    ...
+    &lt;property&gt;
+      &lt;name&gt;hbase.zookeeper.property.clientPort&lt;/name&gt;
+      &lt;value&gt;2222&lt;/value&gt;
+      &lt;description&gt;Property from ZooKeeper's config zoo.cfg.
+      The port at which the clients will connect.
+      &lt;/description&gt;
+    &lt;/property&gt;
+    ...
+    &lt;property&gt;
+      &lt;name&gt;hbase.zookeeper.quorum&lt;/name&gt;
+      &lt;value&gt;rs1.example.com,rs2.example.com,rs3.example.com,rs4.example.com,rs5.example.com&lt;/value&gt;
+      &lt;description&gt;Comma separated list of servers in the ZooKeeper Quorum.
+      For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+      By default this is set to localhost for local and pseudo-distributed modes
+      of operation. For a fully-distributed setup, this should be set to a full
+      list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
+      this is the list of servers which we will start/stop ZooKeeper on.
+      &lt;/description&gt;
+    &lt;/property&gt;
+    ...
+  &lt;/configuration&gt;
+</pre>
+</p>
+<p>
+When HBase manages ZooKeeper, it will start/stop the ZooKeeper servers as a part
+of the regular start/stop scripts. If you would like to run it yourself, you can
+do:
+<pre>
+  ${HBASE_HOME}/bin/hbase-daemons.sh {start,stop} zookeeper
+</pre>
+Note that you can use HBase in this manner to spin up a ZooKeeper cluster,
+unrelated to HBase. Just make sure to set <code>HBASE_MANAGES_ZK</code> to
+<code>false</code> if you want it to stay up so that when HBase shuts down it
+doesn't take ZooKeeper with it.
 </p>
 
 <p>Of note, if you have made <i>HDFS client configuration</i> on your hadoop
cluster, HBase will not

Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/hbase-0.19-two-small-tables.zip
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/hbase-0.19-two-small-tables.zip?rev=794530&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/data/hbase-0.19-two-small-tables.zip
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Thu Jul 16 05:09:53 2009
@@ -53,6 +53,30 @@
     return cols;
   }
 
+  public void testScanSameTimestamp() throws IOException {
+    // returns only 1 of these 2 even though same timestamp
+    KeyValue [] kvs = new KeyValue[] {
+        KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
+    };
+    KeyValueScanner [] scanners = new KeyValueScanner[] {
+        new KeyValueScanFixture(KeyValue.COMPARATOR,
+            kvs)
+    };
+
+    Scan scanSpec = new Scan(Bytes.toBytes("R1"));
+    // this only uses maxVersions (default=1) and TimeRange (default=all)
+    StoreScanner scan =
+        new StoreScanner(scanSpec, CF, Long.MAX_VALUE,
+            KeyValue.COMPARATOR, getCols("a"),
+            scanners);
+
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    assertEquals(true, scan.next(results));
+    assertEquals(1, results.size());
+    assertEquals(kvs[0], results.get(0));
+  }
+
   /**
    * Test test shows exactly how the matcher's return codes confuses the StoreScanner
    * and prevent it from doing the right thing.  Seeking once, then nexting twice

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
Thu Jul 16 05:09:53 2009
@@ -20,14 +20,13 @@
 
 package org.apache.hadoop.hbase.util;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -42,30 +41,32 @@
 import org.apache.hadoop.hbase.client.HConnectionManager;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.ResultScanner;
 
 /**
- * Runs migration of filesystem from hbase 0.x to 0.x
+ * Runs migration of filesystem from hbase 0.19 to hbase 0.20.
+ * Not part of general test suite because takes time.
  */
 public class MigrationTest extends HBaseTestCase {
   private static final Log LOG = LogFactory.getLog(MigrationTest.class);
-  
-  // This is the name of the table that is in the data file.
-  private static final String TABLENAME = "TestUpgrade";
-  
-  // The table has two columns
-  private static final byte [][] TABLENAME_COLUMNS =
-    {Bytes.toBytes("column_a:"), Bytes.toBytes("column_b:")};
 
   // Expected count of rows in migrated table.
-  private static final int EXPECTED_COUNT = 17576;
+  private static final int EXPECTED_COUNT = 3;
 
   /**
-   * Test migration. To be used in future migrations
+   * Test migration.
    * @throws IOException 
    */
-  public void testUpgrade() throws IOException {
+  public void testMigration() throws IOException {
+    Path rootdir = getUnitTestdir(getName());
+    FileSystem fs = FileSystem.get(this.conf);
+    Path hbasedir = loadTestData(fs, rootdir);
+    assertTrue(fs.exists(hbasedir));
+    listPaths(fs, hbasedir, -1);
+    Migrate migrator = new Migrate(this.conf);
+    Path qualified = fs.makeQualified(hbasedir);
+    String uri = qualified.toString();
+    this.conf.set("hbase.rootdir", uri);
+    migrator.run(new String [] {"upgrade"});
   }
   
   /*
@@ -74,24 +75,18 @@
    * @param rootDir
    * @throws IOException
    */
-  private void loadTestData(final FileSystem dfs, final Path rootDir)
+  private Path loadTestData(final FileSystem dfs, final Path rootDir)
   throws IOException {
-    FileSystem localfs = FileSystem.getLocal(conf);
-    // Get path for zip file.  If running this test in eclipse, define
-    // the system property src.testdata for your test run.
-    String srcTestdata = System.getProperty("src.testdata");
-    if (srcTestdata == null) {
-      throw new NullPointerException("Define src.test system property");
-    }
-    Path data = new Path(srcTestdata, "HADOOP-2478-testdata-v0.1.zip");
-    if (!localfs.exists(data)) {
-      throw new FileNotFoundException(data.toString());
-    }
-    FSDataInputStream hs = localfs.open(data);
-    ZipInputStream zip = new ZipInputStream(hs);
-    unzip(zip, dfs, rootDir);
-    zip.close();
-    hs.close();
+    String hbasedir = "hbase-0.19-two-small-tables";
+    InputStream is = this.getClass().getClassLoader().
+      getResourceAsStream("data/" + hbasedir + ".zip");
+    ZipInputStream zip = new ZipInputStream(is);
+    try {
+      unzip(zip, dfs, rootDir);
+    } finally {
+      zip.close();
+    }
+    return new Path(rootDir, hbasedir);
   }
 
   /*
@@ -116,6 +111,7 @@
       assertTrue(hb.isMasterRunning());
       HTableDescriptor [] tables = hb.listTables();
       boolean foundTable = false;
+      /*
       for (int i = 0; i < tables.length; i++) {
         if (Bytes.equals(Bytes.toBytes(TABLENAME), tables[i].getName())) {
           foundTable = true;
@@ -148,6 +144,8 @@
       } finally {
         s.close();
       }
+      
+    */
     } finally {
       HConnectionManager.deleteConnectionInfo(conf, false);
       cluster.shutdown();

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java
(original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java
Thu Jul 16 05:09:53 2009
@@ -25,7 +25,6 @@
 import java.util.Properties;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/master.jsp Thu Jul 16 05:09:53
2009
@@ -28,7 +28,6 @@
   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
-      <meta http-equiv="refresh" content="300"/>
 <title>HBase Master: <%= master.getMasterAddress().getHostname()%>:<%= master.getMasterAddress().getPort()
%></title>
 <link rel="stylesheet" type="text/css" href="/static/hbase.css" />
 <link rel="stylesheet" type="text/css" href="/static/jquery.treeview.css" />

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/master/table.jsp Thu Jul 16 05:09:53 2009
@@ -67,13 +67,12 @@
     %> Compact request accepted. <%
   }
 %>
-<p>This page will refresh in 5 seconds.
+<p>Reload.
 </body>
 <%
 } else {
 %>
 <head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
-      <meta http-equiv="refresh" content="30"/>
 <title>Table: <%= tableName %></title>
 <link rel="stylesheet" type="text/css" href="/static/hbase.css" />
 </head>

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp?rev=794530&r1=794529&r2=794530&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp Thu Jul
16 05:09:53 2009
@@ -20,7 +20,6 @@
   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head><meta http-equiv="Content-Type" content="text/html;charset=UTF-8"/>
-      <meta http-equiv="refresh" content="300"/>
 <title>HBase Region Server: <%= serverInfo.getServerAddress().getHostname() %>:<%=
serverInfo.getServerAddress().getPort() %></title>
 <link rel="stylesheet" type="text/css" href="/static/hbase.css" />
 </head>



Mime
View raw message