hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r574287 [2/2] - in /lucene/hadoop/trunk/src/contrib/hbase: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/io/ src/test/org/apache/hadoop/hbase/
Date Mon, 10 Sep 2007 15:56:18 GMT
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java?rev=574287&r1=574286&r2=574287&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java
Mon Sep 10 08:56:16 2007
@@ -172,11 +172,12 @@
    * @throws IOException
    */
   public void testSplitDeleteOneAddTwoRegions() throws IOException {
+    HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
     // First add a new table.  Its intial region will be added to META region.
     HBaseAdmin admin = new HBaseAdmin(conf);
     Text tableName = new Text(getName());
     admin.createTable(new HTableDescriptor(tableName.toString()));
-    List<HRegionInfo> regions = scan(conf, HConstants.META_TABLE_NAME);
+    List<HRegionInfo> regions = scan(conf, metaTable);
     assertEquals("Expected one region", regions.size(), 1);
     HRegionInfo region = regions.get(0);
     assertTrue("Expected region named for test",
@@ -196,10 +197,10 @@
         homedir, this.conf, null));
     try {
       for (HRegion r : newRegions) {
-        addRegionToMETA(conf, HConstants.META_TABLE_NAME, r,
-            this.cluster.getHMasterAddress(), -1L);
+        addRegionToMETA(conf, metaTable, r, this.cluster.getHMasterAddress(),
+          -1L);
       }
-      regions = scan(conf, HConstants.META_TABLE_NAME);
+      regions = scan(conf, metaTable);
       assertEquals("Should be two regions only", 2, regions.size());
     } finally {
       for (HRegion r : newRegions) {
@@ -209,14 +210,13 @@
     }
   }
   
-  private List<HRegionInfo> scan(final Configuration conf, final Text table)
+  private List<HRegionInfo> scan(final Configuration conf, final HTable t)
   throws IOException {
     List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
     HRegionInterface regionServer = null;
     long scannerId = -1L;
     try {
-      HTable t = new HTable(conf, table);
-      HRegionLocation rl = t.getRegionLocation(table);
+      HRegionLocation rl = t.getRegionLocation(t.getTableName());
       regionServer = t.getConnection().getHRegionConnection(rl.getServerAddress());
       scannerId = regionServer.openScanner(rl.getRegionInfo().getRegionName(),
           HConstants.COLUMN_FAMILY_ARRAY, new Text(),
@@ -263,25 +263,24 @@
   }
   
   private void addRegionToMETA(final Configuration conf,
-      final Text table, final HRegion region,
+      final HTable t, final HRegion region,
       final HServerAddress serverAddress,
       final long startCode)
   throws IOException {
-    HTable t = new HTable(conf, table);
-    try {
-      long lockid = t.startUpdate(region.getRegionName());
-      t.put(lockid, HConstants.COL_REGIONINFO,
-          Writables.getBytes(region.getRegionInfo()));
-      t.put(lockid, HConstants.COL_SERVER,
-        Writables.stringToBytes(serverAddress.toString()));
-      t.put(lockid, HConstants.COL_STARTCODE, Writables.longToBytes(startCode));
-      t.commit(lockid);
-      if (LOG.isDebugEnabled()) {
-        LOG.info("Added region " + region.getRegionName() + " to table " +
-            table);
-      }
-    } finally {
-      t.close();
+    long lockid = t.startUpdate(region.getRegionName());
+    t.put(lockid, HConstants.COL_REGIONINFO,
+      Writables.getBytes(region.getRegionInfo()));
+    t.put(lockid, HConstants.COL_SERVER,
+      Writables.stringToBytes(serverAddress.toString()));
+    t.put(lockid, HConstants.COL_STARTCODE, Writables.longToBytes(startCode));
+    t.commit(lockid);
+    // Assert added.
+    byte [] bytes = t.get(region.getRegionName(), HConstants.COL_REGIONINFO);
+    HRegionInfo hri = Writables.getHRegionInfo(bytes);
+    assertEquals(hri.getRegionId(), region.getRegionId());
+    if (LOG.isDebugEnabled()) {
+      LOG.info("Added region " + region.getRegionName() + " to table " +
+        t.getTableName());
     }
   }
   

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestTimestamp.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestTimestamp.java?rev=574287&r1=574286&r2=574287&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestTimestamp.java
(original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestTimestamp.java
Mon Sep 10 08:56:16 2007
@@ -21,10 +21,14 @@
 import java.io.IOException;
 import java.util.TreeMap;
 
+import org.apache.hadoop.hbase.HColumnDescriptor.CompressionType;
+import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.io.Text;
 
 /**
- * Tests user specifiable time stamps
+ * Tests user specifiable time stamps putting, getting and scanning.  Also
+ * tests same in presence of deletes.  Test cores are written so can be
+ * run against an HRegion and against an HTable: i.e. both local and remote.
  */
 public class TestTimestamp extends HBaseTestCase {
   private static final long T0 = 10L;
@@ -32,74 +36,196 @@
   private static final long T2 = 200L;
   
   private static final String COLUMN_NAME = "contents:";
-  private static final String TABLE_NAME = "test";
-  private static final String VERSION1 = "version1";
-  private static final String LATEST = "latest";
   
   private static final Text COLUMN = new Text(COLUMN_NAME);
-  private static final Text[] COLUMNS = {
-    COLUMN
-  };
-  private static final Text TABLE = new Text(TABLE_NAME);
+  private static final Text[] COLUMNS = {COLUMN};
   private static final Text ROW = new Text("row");
+  
+  // When creating column descriptor, how many versions of a cell to allow.
+  private static final int VERSIONS = 3;
 
   /**
    * Test that delete works according to description in <a
-   * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>
-   * when it comes to timestamps.
+   * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
    * @throws IOException
    */
   public void testDelete() throws IOException {
-    HRegion r = createRegion();
+    final HRegion r = createRegion();
     try {
-      HRegionLoader loader = new HRegionLoader(r);
-      // Add a couple of values for three different timestamps.
-      addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T0);
-      addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T1);
-      addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T2);
-      addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"));
-      // If I delete w/o specifying a timestamp, this means I'm deleting the
-      // latest.
-      delete(r, System.currentTimeMillis());
-      // Verify that I get back T2 through T0.
+     doTestDelete(new HRegionIncommon(r), new FlushCache() {
+      public void flushcache() throws IOException {
+        r.flushcache(false);
+      }
+     });
     } finally {
       r.close();
       r.getLog().closeAndDelete();
     }
   }
-  
-  private void delete(final HRegion r, final long ts) throws IOException {
-    long lockid = r.startUpdate(ROW);
-    r.delete(lockid, COLUMN);
-    r.commit(lockid, ts == -1? System.currentTimeMillis(): ts);
-  }
-  
+
   /**
    * Test scanning against different timestamps.
    * @throws IOException
    */
   public void testTimestampScanning() throws IOException {
-    HRegion r = createRegion();
+    final HRegion r = createRegion();
     try {
-      HRegionLoader loader = new HRegionLoader(r);
-      // Add a couple of values for three different timestamps.
-      addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T0);
-      addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T1);
-      addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"));
-      // Get count of latest items.
-      int count = assertScanContentTimestamp(r, System.currentTimeMillis());
-      // Assert I get same count when I scan at each timestamp.
-      assertEquals(count, assertScanContentTimestamp(r, T0));
-      assertEquals(count, assertScanContentTimestamp(r, T1));
-      // Flush everything out to disk and then retry
-      r.flushcache(false);
-      assertEquals(count, assertScanContentTimestamp(r, T0));
-      assertEquals(count, assertScanContentTimestamp(r, T1));
+      doTestTimestampScanning(new HRegionIncommon(r), new FlushCache() {
+        public void flushcache() throws IOException {
+          r.flushcache(false);
+        }
+       });
     } finally {
       r.close();
       r.getLog().closeAndDelete();
     }
   }
+
+  /**
+   * Basic test of timestamps.
+   * Do the above tests from client side.
+   * @throws IOException
+   */
+  public void testTimestamps() throws IOException {
+    final MiniHBaseCluster cluster = new MiniHBaseCluster(this.conf, 1);
+    try {
+      HTable t = createTable();
+      Incommon incommon = new HTableIncommon(t);
+      doTestDelete(incommon, new FlushCache() {
+        public void flushcache() throws IOException {
+          cluster.flushcache();
+        }
+       });
+      
+      // Perhaps drop and readd the table between tests so the former does
+      // not pollute this latter?  Or put into separate tests.
+      doTestTimestampScanning(incommon, new FlushCache() {
+        public void flushcache() throws IOException {
+          cluster.flushcache();
+        }
+       });
+    } catch (Exception e) {
+      cluster.shutdown();
+    }
+  }
+  
+  /*
+   * Run test that delete works according to description in <a
+   * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
+   * @param incommon
+   * @param flusher
+   * @throws IOException
+   */
+  private void doTestDelete(final Incommon incommon, FlushCache flusher)
+  throws IOException {
+    // Add values at various timestamps (Values are timestampes as bytes).
+    put(incommon, T0);
+    put(incommon, T1);
+    put(incommon, T2);
+    put(incommon);
+    // Verify that returned versions match passed timestamps.
+    assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
+    // If I delete w/o specifying a timestamp, this means I'm deleting the
+    // latest.
+    delete(incommon);
+    // Verify that I get back T2 through T1 -- that the latest version has
+    // been deleted.
+    assertVersions(incommon, new long [] {T2, T1, T0});
+    
+    // Flush everything out to disk and then retry
+    flusher.flushcache();
+    assertVersions(incommon, new long [] {T2, T1, T0});
+    
+    // Now add, back a latest so I can test remove other than the latest.
+    put(incommon);
+    assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
+    delete(incommon, T2);
+    assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
+    // Flush everything out to disk and then retry
+    flusher.flushcache();
+    assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
+    
+    // Now try deleting all from T2 back inclusive (We first need to add T2
+    // back into the mix and to make things a little interesting, delete and
+    // then readd T1.
+    put(incommon, T2);
+    delete(incommon, T1);
+    put(incommon, T1);
+    incommon.deleteAll(ROW, COLUMN, T2);
+    // Should only be current value in set.  Assert this is so
+    assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
+    
+    // Flush everything out to disk and then redo above tests
+    flusher.flushcache();
+    assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP);
+  }
+  
+  private void assertOnlyLatest(final Incommon incommon,
+      final long currentTime)
+  throws IOException {
+    byte [][] bytesBytes = incommon.get(ROW, COLUMN, 3/*Ask for too much*/);
+    assertEquals(1, bytesBytes.length);
+    long time = Writables.bytesToLong(bytesBytes[0]);
+    assertEquals(time, currentTime);
+    assertNull(incommon.get(ROW, COLUMN, T1, 3 /*Too many*/));
+    assertTrue(assertScanContentTimestamp(incommon, T1) == 0);
+  }
+  
+  /*
+   * Assert that returned versions match passed in timestamps and that results
+   * are returned in the right order.  Assert that values when converted to
+   * longs match the corresponding passed timestamp.
+   * @param r
+   * @param tss
+   * @throws IOException
+   */
+  private void assertVersions(final Incommon incommon, final long [] tss)
+  throws IOException {
+    // Assert that 'latest' is what we expect.
+    byte [] bytes = incommon.get(ROW, COLUMN);
+    assertEquals(Writables.bytesToLong(bytes), tss[0]);
+    // Now assert that if we ask for multiple versions, that they come out in
+    // order.
+    byte [][] bytesBytes = incommon.get(ROW, COLUMN, tss.length);
+    assertEquals(bytesBytes.length, tss.length);
+    for (int i = 0; i < bytesBytes.length; i++) {
+      long ts = Writables.bytesToLong(bytesBytes[i]);
+      assertEquals(ts, tss[i]);
+    }
+    // Specify a timestamp get multiple versions.
+    bytesBytes = incommon.get(ROW, COLUMN, tss[0], bytesBytes.length - 1);
+    for (int i = 1; i < bytesBytes.length; i++) {
+      long ts = Writables.bytesToLong(bytesBytes[i]);
+      assertEquals(ts, tss[i]);
+    }
+    // Test scanner returns expected version
+    assertScanContentTimestamp(incommon, tss[0]);
+  }
+  
+  /*
+   * Run test scanning different timestamps.
+   * @param incommon
+   * @param flusher
+   * @throws IOException
+   */
+  private void doTestTimestampScanning(final Incommon incommon,
+      final FlushCache flusher)
+  throws IOException {
+    // Add a couple of values for three different timestamps.
+    put(incommon, T0);
+    put(incommon, T1);
+    put(incommon, HConstants.LATEST_TIMESTAMP);
+    // Get count of latest items.
+    int count = assertScanContentTimestamp(incommon,
+      HConstants.LATEST_TIMESTAMP);
+    // Assert I get same count when I scan at each timestamp.
+    assertEquals(count, assertScanContentTimestamp(incommon, T0));
+    assertEquals(count, assertScanContentTimestamp(incommon, T1));
+    // Flush everything out to disk and then retry
+    flusher.flushcache();
+    assertEquals(count, assertScanContentTimestamp(incommon, T0));
+    assertEquals(count, assertScanContentTimestamp(incommon, T1));
+  }
   
   /*
    * Assert that the scan returns only values < timestamp. 
@@ -108,19 +234,21 @@
    * @return Count of items scanned.
    * @throws IOException
    */
-  private int assertScanContentTimestamp(final HRegion r, final long ts)
+  private int assertScanContentTimestamp(final Incommon in, final long ts)
   throws IOException {
+    HScannerInterface scanner =
+      in.getScanner(COLUMNS, HConstants.EMPTY_START_ROW, ts);
     int count = 0;
-    HInternalScannerInterface scanner =
-      r.getScanner(COLUMNS, HConstants.EMPTY_START_ROW, ts, null);
     try {
       HStoreKey key = new HStoreKey();
       TreeMap<Text, byte []>value = new TreeMap<Text, byte[]>();
       while (scanner.next(key, value)) {
         assertTrue(key.getTimestamp() <= ts);
-        Text row = key.getRow();
-        assertEquals(row.toString(),
-          new String(value.get(COLUMN), HConstants.UTF8_ENCODING));
+        // Content matches the key or HConstants.LATEST_TIMESTAMP.
+        // (Key does not match content if we 'put' with LATEST_TIMESTAMP).
+        long l = Writables.bytesToLong(value.get(COLUMN));
+        assertTrue(key.getTimestamp() == l ||
+          HConstants.LATEST_TIMESTAMP == l);
         count++;
         value.clear();
       }
@@ -129,118 +257,48 @@
     }
     return count;
   }
-
-  /**
-   * Basic test of timestamps.
-   * TODO: Needs rewrite after hadoop-1784 gets fixed.
-   * @throws IOException
-   */
-  public void testTimestamps() throws IOException {
-    MiniHBaseCluster cluster = new MiniHBaseCluster(this.conf, 1);
-    try {
-      HTable table = createTable();
-      
-      // store a value specifying an update time
-      put(table, VERSION1.getBytes(HConstants.UTF8_ENCODING), T0);
-      
-      // store a value specifying 'now' as the update time
-      put(table, LATEST.getBytes(HConstants.UTF8_ENCODING), -1);
-      
-      // delete values older than T1
-      long lockid = table.startUpdate(ROW);
-      table.delete(lockid, COLUMN);
-      table.commit(lockid, T1);
-      
-      // now retrieve...
-      assertGets(table);
-
-      // flush everything out to disk
-      HRegionServer s = cluster.regionThreads.get(0).getRegionServer();
-      for(HRegion r: s.onlineRegions.values() ) {
-        r.flushcache(false);
-      }
-      
-      // now retrieve...
-      assertGets(table);
-      
-      // Test scanners
-      assertScanCount(table, -1, 1);
-      assertScanCount(table, T1, 0);
-    } catch (Exception e) {
-      cluster.shutdown();
-    }
-  }
   
-  /*
-   * Test count of results scanning.
-   * @param table
-   * @param ts
-   * @param expectedCount
-   * @throws IOException
-   */
-  private void assertScanCount(final HTable table, final long ts,
-      final int expectedCount)
+  private void put(final Incommon loader, final long ts)
   throws IOException {
-    HScannerInterface scanner = (ts == -1)?
-      table.obtainScanner(COLUMNS, HConstants.EMPTY_START_ROW):
-      table.obtainScanner(COLUMNS, HConstants.EMPTY_START_ROW, ts);
-    try {
-      HStoreKey key = new HStoreKey();
-      TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
-      int count = 0;
-      while(scanner.next(key, results)) {
-        count++;
-      }
-      assertEquals(count, expectedCount);
-      assertEquals(results.size(), expectedCount);
-      
-    } finally {
-      scanner.close();
-    }
+    put(loader, Writables.longToBytes(ts), ts);
   }
   
-  /*
-   * Test can do basic gets.
-   * Used by testTimestamp above.
-   * @param table
-   * @throws IOException
-   */
-  private void assertGets(final HTable table) throws IOException {
-    // the most recent version:
-    byte[] bytes = table.get(ROW, COLUMN);
-    assertTrue(bytes != null && bytes.length != 0);
-    assertTrue(LATEST.equals(new String(bytes, HConstants.UTF8_ENCODING)));
-    
-    // any version <= time T1
-    byte[][] values = table.get(ROW, COLUMN, T1, 3);
-    assertNull(values);
-    
-    // the version from T0
-    values = table.get(ROW, COLUMN, T0, 3);
-    assertTrue(values.length == 1
-        && VERSION1.equals(new String(values[0], HConstants.UTF8_ENCODING)));
-    
-    // three versions older than now
-    values = table.get(ROW, COLUMN, 3);
-    assertTrue(values.length == 1
-        && LATEST.equals(new String(values[0], HConstants.UTF8_ENCODING)));
+  private void put(final Incommon loader)
+  throws IOException {
+    long ts = HConstants.LATEST_TIMESTAMP;
+    put(loader, Writables.longToBytes(ts), ts);
   }
   
   /*
-   *  Put values.
-   * @param table
+   * Put values.
+   * @param loader
    * @param bytes
    * @param ts
    * @throws IOException
    */
-  private void put(final HTable table, final byte [] bytes, final long ts)
+  private void put(final Incommon loader, final byte [] bytes,
+    final long ts)
   throws IOException {
-    long lockid = table.startUpdate(ROW);
-    table.put(lockid, COLUMN, bytes);
-    if (ts == -1) {
-      table.commit(lockid);
+    long lockid = loader.startBatchUpdate(ROW);
+    loader.put(lockid, COLUMN, bytes);
+    if (ts == HConstants.LATEST_TIMESTAMP) {
+      loader.commit(lockid);
+    } else {
+      loader.commit(lockid, ts);
+    }
+  }
+  
+  private void delete(final Incommon loader) throws IOException {
+    delete(loader, HConstants.LATEST_TIMESTAMP);
+  }
+
+  private void delete(final Incommon loader, final long ts) throws IOException {
+    long lockid = loader.startBatchUpdate(ROW);
+    loader.delete(lockid, COLUMN);
+    if (ts == HConstants.LATEST_TIMESTAMP) {
+      loader.commit(lockid);
     } else {
-      table.commit(lockid, ts);
+      loader.commit(lockid, ts);
     }
   }
   
@@ -250,17 +308,18 @@
    * @throws IOException
    */
   private HTable createTable() throws IOException {
-    HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+    HTableDescriptor desc = new HTableDescriptor(getName());
     desc.addFamily(new HColumnDescriptor(COLUMN_NAME));
     HBaseAdmin admin = new HBaseAdmin(conf);
     admin.createTable(desc);
-    return new HTable(conf, TABLE);
+    return new HTable(conf, new Text(getName()));
   }
   
   private HRegion createRegion() throws IOException {
     HLog hlog = new HLog(this.localFs, this.testDir, this.conf);
     HTableDescriptor htd = createTableDescriptor(getName());
-    htd.addFamily(new HColumnDescriptor(COLUMN_NAME));
+    htd.addFamily(new HColumnDescriptor(COLUMN, VERSIONS,
+      CompressionType.NONE, false, Integer.MAX_VALUE, null));
     HRegionInfo hri = new HRegionInfo(1, htd, null, null);
     return new HRegion(testDir, hlog, this.localFs, this.conf, hri, null);
   }



Mime
View raw message