Return-Path: Delivered-To: apmail-lucene-hadoop-commits-archive@locus.apache.org Received: (qmail 85433 invoked from network); 10 Sep 2007 15:56:50 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 10 Sep 2007 15:56:49 -0000 Received: (qmail 88893 invoked by uid 500); 10 Sep 2007 15:56:43 -0000 Delivered-To: apmail-lucene-hadoop-commits-archive@lucene.apache.org Received: (qmail 88872 invoked by uid 500); 10 Sep 2007 15:56:43 -0000 Mailing-List: contact hadoop-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hadoop-dev@lucene.apache.org Delivered-To: mailing list hadoop-commits@lucene.apache.org Received: (qmail 88863 invoked by uid 99); 10 Sep 2007 15:56:43 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Sep 2007 08:56:43 -0700 X-ASF-Spam-Status: No, hits=-99.1 required=10.0 tests=ALL_TRUSTED,URIBL_RHS_DOB X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Sep 2007 15:58:18 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 4A6CA1A9842; Mon, 10 Sep 2007 08:56:24 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r574287 [2/2] - in /lucene/hadoop/trunk/src/contrib/hbase: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/io/ src/test/org/apache/hadoop/hbase/ Date: Mon, 10 Sep 2007 15:56:18 -0000 To: hadoop-commits@lucene.apache.org From: stack@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20070910155624.4A6CA1A9842@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java?rev=574287&r1=574286&r2=574287&view=diff ============================================================================== --- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java (original) +++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner2.java Mon Sep 10 08:56:16 2007 @@ -172,11 +172,12 @@ * @throws IOException */ public void testSplitDeleteOneAddTwoRegions() throws IOException { + HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME); // First add a new table. Its intial region will be added to META region. HBaseAdmin admin = new HBaseAdmin(conf); Text tableName = new Text(getName()); admin.createTable(new HTableDescriptor(tableName.toString())); - List regions = scan(conf, HConstants.META_TABLE_NAME); + List regions = scan(conf, metaTable); assertEquals("Expected one region", regions.size(), 1); HRegionInfo region = regions.get(0); assertTrue("Expected region named for test", @@ -196,10 +197,10 @@ homedir, this.conf, null)); try { for (HRegion r : newRegions) { - addRegionToMETA(conf, HConstants.META_TABLE_NAME, r, - this.cluster.getHMasterAddress(), -1L); + addRegionToMETA(conf, metaTable, r, this.cluster.getHMasterAddress(), + -1L); } - regions = scan(conf, HConstants.META_TABLE_NAME); + regions = scan(conf, metaTable); assertEquals("Should be two regions only", 2, regions.size()); } finally { for (HRegion r : newRegions) { @@ -209,14 +210,13 @@ } } - private List scan(final Configuration conf, final Text table) + private List scan(final Configuration conf, final HTable t) throws IOException { List regions = new ArrayList(); HRegionInterface regionServer = null; long scannerId = -1L; try { - HTable t = new HTable(conf, table); - HRegionLocation rl = t.getRegionLocation(table); + HRegionLocation rl = t.getRegionLocation(t.getTableName()); regionServer = t.getConnection().getHRegionConnection(rl.getServerAddress()); scannerId = regionServer.openScanner(rl.getRegionInfo().getRegionName(), HConstants.COLUMN_FAMILY_ARRAY, new Text(), @@ -263,25 +263,24 @@ } private void addRegionToMETA(final Configuration conf, - final Text table, final HRegion region, + final HTable t, final HRegion region, final HServerAddress serverAddress, final long startCode) throws IOException { - HTable t = new HTable(conf, table); - try { - long lockid = t.startUpdate(region.getRegionName()); - t.put(lockid, HConstants.COL_REGIONINFO, - Writables.getBytes(region.getRegionInfo())); - t.put(lockid, HConstants.COL_SERVER, - Writables.stringToBytes(serverAddress.toString())); - t.put(lockid, HConstants.COL_STARTCODE, Writables.longToBytes(startCode)); - t.commit(lockid); - if (LOG.isDebugEnabled()) { - LOG.info("Added region " + region.getRegionName() + " to table " + - table); - } - } finally { - t.close(); + long lockid = t.startUpdate(region.getRegionName()); + t.put(lockid, HConstants.COL_REGIONINFO, + Writables.getBytes(region.getRegionInfo())); + t.put(lockid, HConstants.COL_SERVER, + Writables.stringToBytes(serverAddress.toString())); + t.put(lockid, HConstants.COL_STARTCODE, Writables.longToBytes(startCode)); + t.commit(lockid); + // Assert added. + byte [] bytes = t.get(region.getRegionName(), HConstants.COL_REGIONINFO); + HRegionInfo hri = Writables.getHRegionInfo(bytes); + assertEquals(hri.getRegionId(), region.getRegionId()); + if (LOG.isDebugEnabled()) { + LOG.info("Added region " + region.getRegionName() + " to table " + + t.getTableName()); } } Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestTimestamp.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestTimestamp.java?rev=574287&r1=574286&r2=574287&view=diff ============================================================================== --- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestTimestamp.java (original) +++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestTimestamp.java Mon Sep 10 08:56:16 2007 @@ -21,10 +21,14 @@ import java.io.IOException; import java.util.TreeMap; +import org.apache.hadoop.hbase.HColumnDescriptor.CompressionType; +import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.Text; /** - * Tests user specifiable time stamps + * Tests user specifiable time stamps putting, getting and scanning. Also + * tests same in presence of deletes. Test cores are written so can be + * run against an HRegion and against an HTable: i.e. both local and remote. */ public class TestTimestamp extends HBaseTestCase { private static final long T0 = 10L; @@ -32,74 +36,196 @@ private static final long T2 = 200L; private static final String COLUMN_NAME = "contents:"; - private static final String TABLE_NAME = "test"; - private static final String VERSION1 = "version1"; - private static final String LATEST = "latest"; private static final Text COLUMN = new Text(COLUMN_NAME); - private static final Text[] COLUMNS = { - COLUMN - }; - private static final Text TABLE = new Text(TABLE_NAME); + private static final Text[] COLUMNS = {COLUMN}; private static final Text ROW = new Text("row"); + + // When creating column descriptor, how many versions of a cell to allow. + private static final int VERSIONS = 3; /** * Test that delete works according to description in hadoop-1784 - * when it comes to timestamps. + * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784. * @throws IOException */ public void testDelete() throws IOException { - HRegion r = createRegion(); + final HRegion r = createRegion(); try { - HRegionLoader loader = new HRegionLoader(r); - // Add a couple of values for three different timestamps. - addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T0); - addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T1); - addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T2); - addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad")); - // If I delete w/o specifying a timestamp, this means I'm deleting the - // latest. - delete(r, System.currentTimeMillis()); - // Verify that I get back T2 through T0. + doTestDelete(new HRegionIncommon(r), new FlushCache() { + public void flushcache() throws IOException { + r.flushcache(false); + } + }); } finally { r.close(); r.getLog().closeAndDelete(); } } - - private void delete(final HRegion r, final long ts) throws IOException { - long lockid = r.startUpdate(ROW); - r.delete(lockid, COLUMN); - r.commit(lockid, ts == -1? System.currentTimeMillis(): ts); - } - + /** * Test scanning against different timestamps. * @throws IOException */ public void testTimestampScanning() throws IOException { - HRegion r = createRegion(); + final HRegion r = createRegion(); try { - HRegionLoader loader = new HRegionLoader(r); - // Add a couple of values for three different timestamps. - addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T0); - addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad"), T1); - addContent(loader, COLUMN_NAME, START_KEY_BYTES, new Text("aad")); - // Get count of latest items. - int count = assertScanContentTimestamp(r, System.currentTimeMillis()); - // Assert I get same count when I scan at each timestamp. - assertEquals(count, assertScanContentTimestamp(r, T0)); - assertEquals(count, assertScanContentTimestamp(r, T1)); - // Flush everything out to disk and then retry - r.flushcache(false); - assertEquals(count, assertScanContentTimestamp(r, T0)); - assertEquals(count, assertScanContentTimestamp(r, T1)); + doTestTimestampScanning(new HRegionIncommon(r), new FlushCache() { + public void flushcache() throws IOException { + r.flushcache(false); + } + }); } finally { r.close(); r.getLog().closeAndDelete(); } } + + /** + * Basic test of timestamps. + * Do the above tests from client side. + * @throws IOException + */ + public void testTimestamps() throws IOException { + final MiniHBaseCluster cluster = new MiniHBaseCluster(this.conf, 1); + try { + HTable t = createTable(); + Incommon incommon = new HTableIncommon(t); + doTestDelete(incommon, new FlushCache() { + public void flushcache() throws IOException { + cluster.flushcache(); + } + }); + + // Perhaps drop and readd the table between tests so the former does + // not pollute this latter? Or put into separate tests. + doTestTimestampScanning(incommon, new FlushCache() { + public void flushcache() throws IOException { + cluster.flushcache(); + } + }); + } catch (Exception e) { + cluster.shutdown(); + } + } + + /* + * Run test that delete works according to description in hadoop-1784. + * @param incommon + * @param flusher + * @throws IOException + */ + private void doTestDelete(final Incommon incommon, FlushCache flusher) + throws IOException { + // Add values at various timestamps (Values are timestampes as bytes). + put(incommon, T0); + put(incommon, T1); + put(incommon, T2); + put(incommon); + // Verify that returned versions match passed timestamps. + assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1}); + // If I delete w/o specifying a timestamp, this means I'm deleting the + // latest. + delete(incommon); + // Verify that I get back T2 through T1 -- that the latest version has + // been deleted. + assertVersions(incommon, new long [] {T2, T1, T0}); + + // Flush everything out to disk and then retry + flusher.flushcache(); + assertVersions(incommon, new long [] {T2, T1, T0}); + + // Now add, back a latest so I can test remove other than the latest. + put(incommon); + assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1}); + delete(incommon, T2); + assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0}); + // Flush everything out to disk and then retry + flusher.flushcache(); + assertVersions(incommon, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0}); + + // Now try deleting all from T2 back inclusive (We first need to add T2 + // back into the mix and to make things a little interesting, delete and + // then readd T1. + put(incommon, T2); + delete(incommon, T1); + put(incommon, T1); + incommon.deleteAll(ROW, COLUMN, T2); + // Should only be current value in set. Assert this is so + assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP); + + // Flush everything out to disk and then redo above tests + flusher.flushcache(); + assertOnlyLatest(incommon, HConstants.LATEST_TIMESTAMP); + } + + private void assertOnlyLatest(final Incommon incommon, + final long currentTime) + throws IOException { + byte [][] bytesBytes = incommon.get(ROW, COLUMN, 3/*Ask for too much*/); + assertEquals(1, bytesBytes.length); + long time = Writables.bytesToLong(bytesBytes[0]); + assertEquals(time, currentTime); + assertNull(incommon.get(ROW, COLUMN, T1, 3 /*Too many*/)); + assertTrue(assertScanContentTimestamp(incommon, T1) == 0); + } + + /* + * Assert that returned versions match passed in timestamps and that results + * are returned in the right order. Assert that values when converted to + * longs match the corresponding passed timestamp. + * @param r + * @param tss + * @throws IOException + */ + private void assertVersions(final Incommon incommon, final long [] tss) + throws IOException { + // Assert that 'latest' is what we expect. + byte [] bytes = incommon.get(ROW, COLUMN); + assertEquals(Writables.bytesToLong(bytes), tss[0]); + // Now assert that if we ask for multiple versions, that they come out in + // order. + byte [][] bytesBytes = incommon.get(ROW, COLUMN, tss.length); + assertEquals(bytesBytes.length, tss.length); + for (int i = 0; i < bytesBytes.length; i++) { + long ts = Writables.bytesToLong(bytesBytes[i]); + assertEquals(ts, tss[i]); + } + // Specify a timestamp get multiple versions. + bytesBytes = incommon.get(ROW, COLUMN, tss[0], bytesBytes.length - 1); + for (int i = 1; i < bytesBytes.length; i++) { + long ts = Writables.bytesToLong(bytesBytes[i]); + assertEquals(ts, tss[i]); + } + // Test scanner returns expected version + assertScanContentTimestamp(incommon, tss[0]); + } + + /* + * Run test scanning different timestamps. + * @param incommon + * @param flusher + * @throws IOException + */ + private void doTestTimestampScanning(final Incommon incommon, + final FlushCache flusher) + throws IOException { + // Add a couple of values for three different timestamps. + put(incommon, T0); + put(incommon, T1); + put(incommon, HConstants.LATEST_TIMESTAMP); + // Get count of latest items. + int count = assertScanContentTimestamp(incommon, + HConstants.LATEST_TIMESTAMP); + // Assert I get same count when I scan at each timestamp. + assertEquals(count, assertScanContentTimestamp(incommon, T0)); + assertEquals(count, assertScanContentTimestamp(incommon, T1)); + // Flush everything out to disk and then retry + flusher.flushcache(); + assertEquals(count, assertScanContentTimestamp(incommon, T0)); + assertEquals(count, assertScanContentTimestamp(incommon, T1)); + } /* * Assert that the scan returns only values < timestamp. @@ -108,19 +234,21 @@ * @return Count of items scanned. * @throws IOException */ - private int assertScanContentTimestamp(final HRegion r, final long ts) + private int assertScanContentTimestamp(final Incommon in, final long ts) throws IOException { + HScannerInterface scanner = + in.getScanner(COLUMNS, HConstants.EMPTY_START_ROW, ts); int count = 0; - HInternalScannerInterface scanner = - r.getScanner(COLUMNS, HConstants.EMPTY_START_ROW, ts, null); try { HStoreKey key = new HStoreKey(); TreeMapvalue = new TreeMap(); while (scanner.next(key, value)) { assertTrue(key.getTimestamp() <= ts); - Text row = key.getRow(); - assertEquals(row.toString(), - new String(value.get(COLUMN), HConstants.UTF8_ENCODING)); + // Content matches the key or HConstants.LATEST_TIMESTAMP. + // (Key does not match content if we 'put' with LATEST_TIMESTAMP). + long l = Writables.bytesToLong(value.get(COLUMN)); + assertTrue(key.getTimestamp() == l || + HConstants.LATEST_TIMESTAMP == l); count++; value.clear(); } @@ -129,118 +257,48 @@ } return count; } - - /** - * Basic test of timestamps. - * TODO: Needs rewrite after hadoop-1784 gets fixed. - * @throws IOException - */ - public void testTimestamps() throws IOException { - MiniHBaseCluster cluster = new MiniHBaseCluster(this.conf, 1); - try { - HTable table = createTable(); - - // store a value specifying an update time - put(table, VERSION1.getBytes(HConstants.UTF8_ENCODING), T0); - - // store a value specifying 'now' as the update time - put(table, LATEST.getBytes(HConstants.UTF8_ENCODING), -1); - - // delete values older than T1 - long lockid = table.startUpdate(ROW); - table.delete(lockid, COLUMN); - table.commit(lockid, T1); - - // now retrieve... - assertGets(table); - - // flush everything out to disk - HRegionServer s = cluster.regionThreads.get(0).getRegionServer(); - for(HRegion r: s.onlineRegions.values() ) { - r.flushcache(false); - } - - // now retrieve... - assertGets(table); - - // Test scanners - assertScanCount(table, -1, 1); - assertScanCount(table, T1, 0); - } catch (Exception e) { - cluster.shutdown(); - } - } - /* - * Test count of results scanning. - * @param table - * @param ts - * @param expectedCount - * @throws IOException - */ - private void assertScanCount(final HTable table, final long ts, - final int expectedCount) + private void put(final Incommon loader, final long ts) throws IOException { - HScannerInterface scanner = (ts == -1)? - table.obtainScanner(COLUMNS, HConstants.EMPTY_START_ROW): - table.obtainScanner(COLUMNS, HConstants.EMPTY_START_ROW, ts); - try { - HStoreKey key = new HStoreKey(); - TreeMap results = new TreeMap(); - int count = 0; - while(scanner.next(key, results)) { - count++; - } - assertEquals(count, expectedCount); - assertEquals(results.size(), expectedCount); - - } finally { - scanner.close(); - } + put(loader, Writables.longToBytes(ts), ts); } - /* - * Test can do basic gets. - * Used by testTimestamp above. - * @param table - * @throws IOException - */ - private void assertGets(final HTable table) throws IOException { - // the most recent version: - byte[] bytes = table.get(ROW, COLUMN); - assertTrue(bytes != null && bytes.length != 0); - assertTrue(LATEST.equals(new String(bytes, HConstants.UTF8_ENCODING))); - - // any version <= time T1 - byte[][] values = table.get(ROW, COLUMN, T1, 3); - assertNull(values); - - // the version from T0 - values = table.get(ROW, COLUMN, T0, 3); - assertTrue(values.length == 1 - && VERSION1.equals(new String(values[0], HConstants.UTF8_ENCODING))); - - // three versions older than now - values = table.get(ROW, COLUMN, 3); - assertTrue(values.length == 1 - && LATEST.equals(new String(values[0], HConstants.UTF8_ENCODING))); + private void put(final Incommon loader) + throws IOException { + long ts = HConstants.LATEST_TIMESTAMP; + put(loader, Writables.longToBytes(ts), ts); } /* - * Put values. - * @param table + * Put values. + * @param loader * @param bytes * @param ts * @throws IOException */ - private void put(final HTable table, final byte [] bytes, final long ts) + private void put(final Incommon loader, final byte [] bytes, + final long ts) throws IOException { - long lockid = table.startUpdate(ROW); - table.put(lockid, COLUMN, bytes); - if (ts == -1) { - table.commit(lockid); + long lockid = loader.startBatchUpdate(ROW); + loader.put(lockid, COLUMN, bytes); + if (ts == HConstants.LATEST_TIMESTAMP) { + loader.commit(lockid); + } else { + loader.commit(lockid, ts); + } + } + + private void delete(final Incommon loader) throws IOException { + delete(loader, HConstants.LATEST_TIMESTAMP); + } + + private void delete(final Incommon loader, final long ts) throws IOException { + long lockid = loader.startBatchUpdate(ROW); + loader.delete(lockid, COLUMN); + if (ts == HConstants.LATEST_TIMESTAMP) { + loader.commit(lockid); } else { - table.commit(lockid, ts); + loader.commit(lockid, ts); } } @@ -250,17 +308,18 @@ * @throws IOException */ private HTable createTable() throws IOException { - HTableDescriptor desc = new HTableDescriptor(TABLE_NAME); + HTableDescriptor desc = new HTableDescriptor(getName()); desc.addFamily(new HColumnDescriptor(COLUMN_NAME)); HBaseAdmin admin = new HBaseAdmin(conf); admin.createTable(desc); - return new HTable(conf, TABLE); + return new HTable(conf, new Text(getName())); } private HRegion createRegion() throws IOException { HLog hlog = new HLog(this.localFs, this.testDir, this.conf); HTableDescriptor htd = createTableDescriptor(getName()); - htd.addFamily(new HColumnDescriptor(COLUMN_NAME)); + htd.addFamily(new HColumnDescriptor(COLUMN, VERSIONS, + CompressionType.NONE, false, Integer.MAX_VALUE, null)); HRegionInfo hri = new HRegionInfo(1, htd, null, null); return new HRegion(testDir, hlog, this.localFs, this.conf, hri, null); }