Return-Path: Delivered-To: apmail-hadoop-hbase-commits-archive@minotaur.apache.org Received: (qmail 31729 invoked from network); 7 May 2010 19:28:29 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 7 May 2010 19:28:29 -0000 Received: (qmail 53990 invoked by uid 500); 7 May 2010 19:28:29 -0000 Delivered-To: apmail-hadoop-hbase-commits-archive@hadoop.apache.org Received: (qmail 53957 invoked by uid 500); 7 May 2010 19:28:29 -0000 Mailing-List: contact hbase-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hbase-dev@hadoop.apache.org Delivered-To: mailing list hbase-commits@hadoop.apache.org Received: (qmail 53950 invoked by uid 99); 7 May 2010 19:28:29 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 May 2010 19:28:29 +0000 X-ASF-Spam-Status: No, hits=-1678.6 required=10.0 tests=ALL_TRUSTED,AWL,T_FRT_ADULT2 X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 May 2010 19:28:20 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id E8D532388B46; Fri, 7 May 2010 19:26:56 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r942186 [12/18] - in /hadoop/hbase/trunk: ./ contrib/stargate/core/src/test/java/org/apache/hadoop/hbase/stargate/ core/src/main/java/org/apache/hadoop/hbase/ core/src/main/java/org/apache/hadoop/hbase/client/ core/src/main/java/org/apache/... Date: Fri, 07 May 2010 19:26:51 -0000 To: hbase-commits@hadoop.apache.org From: stack@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100507192656.E8D532388B46@eris.apache.org> Modified: hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/util/Writables.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/util/Writables.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/util/Writables.java (original) +++ hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/util/Writables.java Fri May 7 19:26:45 2010 @@ -35,7 +35,7 @@ import java.io.IOException; public class Writables { /** * @param w writable - * @return The bytes of w gotten by running its + * @return The bytes of w gotten by running its * {@link Writable#write(java.io.DataOutput)} method. * @throws IOException e * @see #getWritable(byte[], Writable) @@ -118,7 +118,7 @@ public class Writables { throws IOException { return (HRegionInfo)getWritable(bytes, new HRegionInfo()); } - + /** * @param bytes serialized bytes * @return A HRegionInfo instance built out of passed bytes Modified: hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java (original) +++ hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java Fri May 7 19:26:45 2010 @@ -103,7 +103,7 @@ public class ZooKeeperWrapper implements "master"); String stateZNodeName = conf.get("zookeeper.znode.state", "shutdown"); - + rootRegionZNode = getZNode(parentZNode, rootServerZNodeName); rsZNode = getZNode(parentZNode, rsZNodeName); masterElectionZNode = getZNode(parentZNode, masterAddressZNodeName); @@ -193,31 +193,31 @@ public class ZooKeeperWrapper implements } return sb.toString(); } - + /** * Gets the statistics from the given server. Uses a 1 minute timeout. - * + * * @param server The server to get the statistics from. * @return The array of response strings. * @throws IOException When the socket communication fails. */ - public String[] getServerStats(String server) + public String[] getServerStats(String server) throws IOException { return getServerStats(server, 60 * 1000); } - + /** * Gets the statistics from the given server. - * + * * @param server The server to get the statistics from. * @param timeout The socket timeout to use. * @return The array of response strings. * @throws IOException When the socket communication fails. */ - public String[] getServerStats(String server, int timeout) + public String[] getServerStats(String server, int timeout) throws IOException { String[] sp = server.split(":"); - Socket socket = new Socket(sp[0], + Socket socket = new Socket(sp[0], sp.length > 1 ? Integer.parseInt(sp[1]) : 2181); socket.setSoTimeout(timeout); PrintWriter out = new PrintWriter(socket.getOutputStream(), true); @@ -305,7 +305,7 @@ public class ZooKeeperWrapper implements public HServerAddress readMasterAddress(Watcher watcher) { return readAddress(masterElectionZNode, watcher); } - + /** * Watch the state of the cluster, up or down * @param watcher Watcher to set on cluster state node @@ -319,7 +319,7 @@ public class ZooKeeperWrapper implements LOG.warn("Failed to check on ZNode " + clusterStateZNode, e); } } - + /** * Set the cluster state, up or down * @param up True to write the node, false to delete it @@ -332,7 +332,7 @@ public class ZooKeeperWrapper implements try { if(up) { byte[] data = Bytes.toBytes("up"); - zooKeeper.create(clusterStateZNode, data, + zooKeeper.create(clusterStateZNode, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); LOG.debug("State node wrote in ZooKeeper"); } else { @@ -579,7 +579,7 @@ public class ZooKeeperWrapper implements return false; } - + /** * Scans the regions servers directory * @return A list of server addresses @@ -587,7 +587,7 @@ public class ZooKeeperWrapper implements public List scanRSDirectory() { return scanAddressDirectory(rsZNode, null); } - + /** * Method used to make sure the region server directory is empty. * @@ -605,7 +605,7 @@ public class ZooKeeperWrapper implements LOG.warn("Failed to delete " + rsZNode + " znodes in ZooKeeper: " + e); } } - + private boolean checkExistenceOf(String path) { Stat stat = null; try { @@ -630,7 +630,7 @@ public class ZooKeeperWrapper implements LOG.warn("Failed to close connection with ZooKeeper"); } } - + public String getZNode(String parentZNode, String znodeName) { return znodeName.charAt(0) == ZNODE_PATH_SEPARATOR ? znodeName : joinPath(parentZNode, znodeName); @@ -731,6 +731,6 @@ public class ZooKeeperWrapper implements return conf.get(ZOOKEEPER_QUORUM)+":"+ conf.get(ZOOKEEPER_ZNODE_PARENT); } - - + + } Modified: hadoop/hbase/trunk/core/src/main/resources/hbase-default.xml URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/main/resources/hbase-default.xml?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/main/resources/hbase-default.xml (original) +++ hadoop/hbase/trunk/core/src/main/resources/hbase-default.xml Fri May 7 19:26:45 2010 @@ -66,7 +66,7 @@ 2097152 Size of the write buffer in bytes. A bigger buffer takes more memory -- on both the client and server side since server instantiates - the passed write buffer to process it -- but reduces the number of RPC. + the passed write buffer to process it -- but reduces the number of RPC. For an estimate of server-side memory-used, evaluate hbase.client.write.buffer * hbase.regionserver.handler.count @@ -143,7 +143,7 @@ instance. This is to set an upper boundary for a single entry saved in a storage file. Since they cannot be split it helps avoiding that a region cannot be split any further because the data is too large. It seems wise - to set this to a fraction of the maximum region size. Setting it to zero + to set this to a fraction of the maximum region size. Setting it to zero or less disables the check. @@ -231,7 +231,7 @@ hbase.regionserver.dns.interface default - The name of the Network Interface from which a region server + The name of the Network Interface from which a region server should report its IP address. @@ -246,7 +246,7 @@ hbase.master.dns.interface default - The name of the Network Interface from which a master + The name of the Network Interface from which a master should report its IP address. @@ -254,14 +254,14 @@ hbase.master.dns.nameserver default The host name or IP address of the name server (DNS) - which a master should use to determine the host name used + which a master should use to determine the host name used for communication and display purposes. hbase.regionserver.global.memstore.upperLimit 0.4 - Maximum size of all memstores in a region server before new + Maximum size of all memstores in a region server before new updates are blocked and flushes are forced. Defaults to 40% of heap @@ -269,12 +269,12 @@ hbase.regionserver.global.memstore.lowerLimit 0.35 When memstores are being forced to flush to make room in - memory, keep flushing until we hit this mark. Defaults to 30% of heap. + memory, keep flushing until we hit this mark. Defaults to 30% of heap. This value equal to hbase.regionserver.global.memstore.upperLimit causes - the minimum possible flushing to occur when updates are blocked due to + the minimum possible flushing to occur when updates are blocked due to memstore limiting. - + hbase.hbasemaster.maxregionopen 120000 @@ -359,7 +359,7 @@ During a compaction, updates cannot be flushed to disk. Long compactions require memory sufficient to carry the logging of all updates across the duration of the compaction. - + If too large, clients timeout during compaction. Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/AbstractMergeTestBase.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/AbstractMergeTestBase.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/AbstractMergeTestBase.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/AbstractMergeTestBase.java Fri May 7 19:26:45 2010 @@ -39,27 +39,27 @@ public abstract class AbstractMergeTestB protected HTableDescriptor desc; protected ImmutableBytesWritable value; protected boolean startMiniHBase; - + public AbstractMergeTestBase() { this(true); } - - /** constructor + + /** constructor * @param startMiniHBase */ public AbstractMergeTestBase(boolean startMiniHBase) { super(); - + this.startMiniHBase = startMiniHBase; - + // We will use the same value for the rows as that is not really important here - + String partialValue = String.valueOf(System.currentTimeMillis()); StringBuilder val = new StringBuilder(); while(val.length() < 1024) { val.append(partialValue); } - + try { value = new ImmutableBytesWritable( val.toString().getBytes(HConstants.UTF8_ENCODING)); @@ -81,10 +81,10 @@ public abstract class AbstractMergeTestB public void preHBaseClusterSetup() throws Exception { conf.setLong("hbase.hregion.max.filesize", 64L * 1024L * 1024L); - // We create three data regions: The first is too large to merge since it - // will be > 64 MB in size. The second two will be smaller and will be + // We create three data regions: The first is too large to merge since it + // will be > 64 MB in size. The second two will be smaller and will be // selected for merging. - + // To ensure that the first region is larger than 64MB we need to write at // least 65536 rows. We will make certain by writing 70000 @@ -103,12 +103,12 @@ public abstract class AbstractMergeTestB createAregion(row_70001, row_80001, 70001, 10000), createAregion(row_80001, null, 80001, 11000) }; - + // Now create the root and meta regions and insert the data regions // created above into the meta createRootAndMetaRegions(); - + for(int i = 0; i < regions.length; i++) { HRegion.addRegionToMETA(meta, regions[i]); } @@ -118,9 +118,9 @@ public abstract class AbstractMergeTestB private HRegion createAregion(byte [] startKey, byte [] endKey, int firstRow, int nrows) throws IOException { - + HRegion region = createNewHRegion(desc, startKey, endKey); - + System.out.println("created region " + Bytes.toString(region.getRegionName())); Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseClusterTestCase.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseClusterTestCase.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseClusterTestCase.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseClusterTestCase.java Fri May 7 19:26:45 2010 @@ -51,22 +51,22 @@ public abstract class HBaseClusterTestCa public HBaseClusterTestCase() { this(1); } - + /** * Start a MiniHBaseCluster with regionServers region servers in-process to * start with. Also, start a MiniDfsCluster before starting the hbase cluster. * The configuration used will be edited so that this works correctly. * @param regionServers number of region servers to start. - */ + */ public HBaseClusterTestCase(int regionServers) { this(regionServers, true); } - + /** in-process to * start with. Optionally, startDfs indicates if a MiniDFSCluster should be * started. If startDfs is false, the assumption is that an external DFS is * configured in hbase-site.xml and is already started, or you have started a - * MiniDFSCluster on your own and edited the configuration in memory. (You + * MiniDFSCluster on your own and edited the configuration in memory. (You * can modify the config used by overriding the preHBaseClusterSetup method.) * @param regionServers number of region servers to start. * @param startDfs set to true if MiniDFS should be started @@ -83,12 +83,12 @@ public abstract class HBaseClusterTestCa /** * Subclass hook. - * + * * Run after dfs is ready but before hbase cluster is started up. */ protected void preHBaseClusterSetup() throws Exception { // continue - } + } /** * Actually start the MiniHBase instance. @@ -110,13 +110,13 @@ public abstract class HBaseClusterTestCa new HTable(conf, HConstants.META_TABLE_NAME); } } - + /** * Run after hbase cluster is started up. */ protected void postHBaseClusterSetup() throws Exception { // continue - } + } @Override protected void setUp() throws Exception { @@ -139,9 +139,9 @@ public abstract class HBaseClusterTestCa // do the super setup now. if we had done it first, then we would have // gotten our conf all mangled and a local fs started up. super.setUp(); - + // run the pre-cluster setup - preHBaseClusterSetup(); + preHBaseClusterSetup(); // start the instance hBaseClusterSetup(); @@ -194,7 +194,7 @@ public abstract class HBaseClusterTestCa // "Temporary end-of-test thread dump debugging HADOOP-2040: " + getName()); } - + /** * Use this utility method debugging why cluster won't go down. On a * period it throws a thread dump. Method ends when all cluster Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java Fri May 7 19:26:45 2010 @@ -72,11 +72,11 @@ public abstract class HBaseTestCase exte protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR}; protected String START_KEY; protected static final int MAXVERSIONS = 3; - + static { initialize(); } - + public volatile HBaseConfiguration conf; /** constructor */ @@ -84,7 +84,7 @@ public abstract class HBaseTestCase exte super(); init(); } - + /** * @param name */ @@ -92,7 +92,7 @@ public abstract class HBaseTestCase exte super(name); init(); } - + private void init() { conf = new HBaseConfiguration(); try { @@ -102,7 +102,7 @@ public abstract class HBaseTestCase exte fail(); } } - + /** * Note that this method must be called after the mini hdfs cluster has * started or we end up with a local file system. @@ -131,7 +131,7 @@ public abstract class HBaseTestCase exte throw e; } } - + @Override protected void tearDown() throws Exception { try { @@ -158,11 +158,11 @@ public abstract class HBaseTestCase exte Path rootdir = filesystem.makeQualified( new Path(conf.get(HConstants.HBASE_DIR))); filesystem.mkdirs(rootdir); - + return HRegion.createHRegion(new HRegionInfo(desc, startKey, endKey), rootdir, conf); } - + protected HRegion openClosedRegion(final HRegion closedRegion) throws IOException { HRegion r = new HRegion(closedRegion.getBaseDir(), closedRegion.getLog(), @@ -171,7 +171,7 @@ public abstract class HBaseTestCase exte r.initialize(null, null); return r; } - + /** * Create a table of name name with {@link COLUMNS} for * families. @@ -181,7 +181,7 @@ public abstract class HBaseTestCase exte protected HTableDescriptor createTableDescriptor(final String name) { return createTableDescriptor(name, MAXVERSIONS); } - + /** * Create a table of name name with {@link COLUMNS} for * families. @@ -204,7 +204,7 @@ public abstract class HBaseTestCase exte false, HConstants.REPLICATION_SCOPE_LOCAL)); return htd; } - + /** * Add content to region r on the passed column * column. @@ -261,13 +261,13 @@ public abstract class HBaseTestCase exte throws IOException { return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1); } - + protected static long addContent(final Incommon updater, final String family, final String column, final byte [] startKeyBytes, final byte [] endKey) throws IOException { return addContent(updater, family, column, startKeyBytes, endKey, -1); } - + /** * Add content to region r on the passed column * column. @@ -337,7 +337,7 @@ public abstract class HBaseTestCase exte } catch (IOException ex) { ex.printStackTrace(); throw ex; - } + } } catch (RuntimeException ex) { ex.printStackTrace(); throw ex; @@ -353,7 +353,7 @@ public abstract class HBaseTestCase exte } return count; } - + /** * Implementors can flushcache. */ @@ -363,16 +363,16 @@ public abstract class HBaseTestCase exte */ public void flushcache() throws IOException; } - + /** * Interface used by tests so can do common operations against an HTable * or an HRegion. - * + * * TOOD: Come up w/ a better name for this interface. */ public static interface Incommon { /** - * + * * @param delete * @param lockid * @param writeToWAL @@ -388,7 +388,7 @@ public abstract class HBaseTestCase exte public void put(Put put) throws IOException; public Result get(Get get) throws IOException; - + /** * @param family * @param qualifiers @@ -401,35 +401,35 @@ public abstract class HBaseTestCase exte byte [] firstRow, long ts) throws IOException; } - + /** * A class that makes a {@link Incommon} out of a {@link HRegion} */ public static class HRegionIncommon implements Incommon, FlushCache { final HRegion region; - + /** * @param HRegion */ public HRegionIncommon(final HRegion HRegion) { this.region = HRegion; } - + public void put(Put put) throws IOException { region.put(put); } - + public void delete(Delete delete, Integer lockid, boolean writeToWAL) throws IOException { this.region.delete(delete, lockid, writeToWAL); } - + public Result get(Get get) throws IOException { return region.get(get, null); } - + public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers, - byte [] firstRow, long ts) + byte [] firstRow, long ts) throws IOException { Scan scan = new Scan(firstRow); if(qualifiers == null || qualifiers.length == 0) { @@ -440,14 +440,14 @@ public abstract class HBaseTestCase exte } } scan.setTimeRange(0, ts); - return new + return new InternalScannerIncommon(region.getScanner(scan)); } - + public Result get(Get get, Integer lockid) throws IOException{ return this.region.get(get, lockid); } - + public void flushcache() throws IOException { this.region.flushcache(); @@ -467,23 +467,23 @@ public abstract class HBaseTestCase exte super(); this.table = table; } - + public void put(Put put) throws IOException { table.put(put); } - - + + public void delete(Delete delete, Integer lockid, boolean writeToWAL) throws IOException { this.table.delete(delete); } - + public Result get(Get get) throws IOException { return table.get(get); } public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers, - byte [] firstRow, long ts) + byte [] firstRow, long ts) throws IOException { Scan scan = new Scan(firstRow); if(qualifiers == null || qualifiers.length == 0) { @@ -494,25 +494,25 @@ public abstract class HBaseTestCase exte } } scan.setTimeRange(0, ts); - return new + return new ClientScannerIncommon(table.getScanner(scan)); } } - - public interface ScannerIncommon + + public interface ScannerIncommon extends Iterable { public boolean next(List values) throws IOException; - + public void close() throws IOException; } - + public static class ClientScannerIncommon implements ScannerIncommon { ResultScanner scanner; public ClientScannerIncommon(ResultScanner scanner) { this.scanner = scanner; } - + public boolean next(List values) throws IOException { Result results = scanner.next(); @@ -523,38 +523,38 @@ public abstract class HBaseTestCase exte values.addAll(results.list()); return true; } - + public void close() throws IOException { scanner.close(); } - + @SuppressWarnings("unchecked") public Iterator iterator() { return scanner.iterator(); } } - + public static class InternalScannerIncommon implements ScannerIncommon { InternalScanner scanner; - + public InternalScannerIncommon(InternalScanner scanner) { this.scanner = scanner; } - + public boolean next(List results) throws IOException { return scanner.next(results); } - + public void close() throws IOException { scanner.close(); } - + public Iterator iterator() { throw new UnsupportedOperationException(); } } - + // protected void assertCellEquals(final HRegion region, final byte [] row, // final byte [] column, final long timestamp, final String value) // throws IOException { @@ -565,11 +565,11 @@ public abstract class HBaseTestCase exte // cell_value); // } else { // if (cell_value == null) { -// fail(Bytes.toString(column) + " at timestamp " + timestamp + +// fail(Bytes.toString(column) + " at timestamp " + timestamp + // "\" was expected to be \"" + value + " but was null"); // } // if (cell_value != null) { -// assertEquals(Bytes.toString(column) + " at timestamp " +// assertEquals(Bytes.toString(column) + " at timestamp " // + timestamp, value, new String(cell_value.getValue())); // } // } @@ -582,30 +582,30 @@ public abstract class HBaseTestCase exte Get get = new Get(row); get.setTimeStamp(timestamp); Result res = region.get(get, null); - NavigableMap>> map = + NavigableMap>> map = res.getMap(); byte [] res_value = map.get(family).get(qualifier).get(timestamp); - + if (value == null) { assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + " at timestamp " + timestamp, null, res_value); } else { if (res_value == null) { - fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) + - " at timestamp " + timestamp + "\" was expected to be \"" + + fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) + + " at timestamp " + timestamp + "\" was expected to be \"" + Bytes.toStringBinary(value) + " but was null"); } if (res_value != null) { assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + - " at timestamp " + + " at timestamp " + timestamp, value, new String(res_value)); } } } - + /** * Initializes parameters used in the test environment: - * + * * Sets the configuration parameter TEST_DIRECTORY_KEY if not already set. * Sets the boolean debugging if "DEBUGGING" is set in the environment. * If debugging is enabled, reconfigures logging so that the root log level is @@ -620,7 +620,7 @@ public abstract class HBaseTestCase exte /** * Common method to close down a MiniDFSCluster and the associated file system - * + * * @param cluster */ public static void shutdownDfs(MiniDFSCluster cluster) { @@ -645,14 +645,14 @@ public abstract class HBaseTestCase exte } } } - + protected void createRootAndMetaRegions() throws IOException { root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir, conf); - meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, + meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf); HRegion.addRegionToMETA(root, meta); } - + protected void closeRootAndMeta() throws IOException { if (meta != null) { meta.close(); Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Fri May 7 19:26:45 2010 @@ -138,7 +138,7 @@ public class HBaseTestingUtility { /** * Start up a minicluster of hbase, dfs, and zookeeper. - * @throws Exception + * @throws Exception */ public void startMiniCluster() throws Exception { startMiniCluster(1); @@ -148,7 +148,7 @@ public class HBaseTestingUtility { * Call this if you only want a zk cluster. * @see #startMiniZKCluster() if you want zk + dfs + hbase mini cluster. * @throws Exception - * @see #shutdownMiniZKCluster() + * @see #shutdownMiniZKCluster() */ public void startMiniZKCluster() throws Exception { isRunningCluster(); @@ -169,7 +169,7 @@ public class HBaseTestingUtility { * @see #startMiniZKCluster() */ public void shutdownMiniZKCluster() throws IOException { - if (this.zkCluster != null) this.zkCluster.shutdown(); + if (this.zkCluster != null) this.zkCluster.shutdown(); } /** @@ -203,7 +203,7 @@ public class HBaseTestingUtility { // the TEST_DIRECTORY_KEY to make bad blocks, a feature we are not using, // but otherwise, just in constructor. System.setProperty(TEST_DIRECTORY_KEY, oldBuildTestDir); - + // Mangle conf so fs parameter points to minidfs we just started up FileSystem fs = this.dfsCluster.getFileSystem(); this.conf.set("fs.defaultFS", fs.getUri().toString()); @@ -279,7 +279,7 @@ public class HBaseTestingUtility { * @return An HTable instance for the created table. * @throws IOException */ - public HTable createTable(byte[] tableName, byte[] family) + public HTable createTable(byte[] tableName, byte[] family) throws IOException{ return createTable(tableName, new byte[][]{family}); } @@ -291,7 +291,7 @@ public class HBaseTestingUtility { * @return An HTable instance for the created table. * @throws IOException */ - public HTable createTable(byte[] tableName, byte[][] families) + public HTable createTable(byte[] tableName, byte[][] families) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); for(byte[] family : families) { @@ -408,16 +408,16 @@ public class HBaseTestingUtility { } return rowCount; } - + /** * Creates many regions names "aaa" to "zzz". - * + * * @param table The table to use for the data. * @param columnFamily The family to insert the data into. * @return count of regions created. * @throws IOException When creating the regions fails. */ - public int createMultiRegions(HTable table, byte[] columnFamily) + public int createMultiRegions(HTable table, byte[] columnFamily) throws IOException { return createMultiRegions(getConfiguration(), table, columnFamily); } @@ -431,11 +431,11 @@ public class HBaseTestingUtility { * @throws IOException When creating the regions fails. */ public int createMultiRegions(final Configuration c, final HTable table, - final byte[] columnFamily) + final byte[] columnFamily) throws IOException { byte[][] KEYS = { HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("bbb"), - Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"), + Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), Bytes.toBytes("eee"), Bytes.toBytes("fff"), Bytes.toBytes("ggg"), Bytes.toBytes("hhh"), Bytes.toBytes("iii"), Bytes.toBytes("jjj"), Bytes.toBytes("kkk"), Bytes.toBytes("lll"), Bytes.toBytes("mmm"), Bytes.toBytes("nnn"), @@ -452,18 +452,18 @@ public class HBaseTestingUtility { htd.addFamily(hcd); } // remove empty region - this is tricky as the mini cluster during the test - // setup already has the ",,123456789" row with an empty start - // and end key. Adding the custom regions below adds those blindly, - // including the new start region from empty to "bbb". lg + // setup already has the ",,123456789" row with an empty start + // and end key. Adding the custom regions below adds those blindly, + // including the new start region from empty to "bbb". lg List rows = getMetaTableRows(); // add custom ones int count = 0; for (int i = 0; i < KEYS.length; i++) { int j = (i + 1) % KEYS.length; - HRegionInfo hri = new HRegionInfo(table.getTableDescriptor(), + HRegionInfo hri = new HRegionInfo(table.getTableDescriptor(), KEYS[i], KEYS[j]); Put put = new Put(hri.getRegionName()); - put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(hri)); meta.put(put); LOG.info("createMultiRegions: inserted " + hri.toString()); @@ -471,7 +471,7 @@ public class HBaseTestingUtility { } // see comment above, remove "old" (or previous) single region for (byte[] row : rows) { - LOG.info("createMultiRegions: deleting meta row -> " + + LOG.info("createMultiRegions: deleting meta row -> " + Bytes.toStringBinary(row)); meta.delete(new Delete(row)); } @@ -491,7 +491,7 @@ public class HBaseTestingUtility { List rows = new ArrayList(); ResultScanner s = t.getScanner(new Scan()); for (Result result : s) { - LOG.info("getMetaTableRows: row -> " + + LOG.info("getMetaTableRows: row -> " + Bytes.toStringBinary(result.getRow())); rows.add(result.getRow()); } @@ -509,7 +509,7 @@ public class HBaseTestingUtility { ArrayList deletes = new ArrayList(); ResultScanner s = t.getScanner(new Scan()); for (Result result : s) { - LOG.info("emptyMetaTable: remove row -> " + + LOG.info("emptyMetaTable: remove row -> " + Bytes.toStringBinary(result.getRow())); Delete del = new Delete(result.getRow()); deletes.add(del); @@ -517,9 +517,9 @@ public class HBaseTestingUtility { s.close(); t.delete(deletes); } - + /** - * Starts a MiniMRCluster with a default number of + * Starts a MiniMRCluster with a default number of * TaskTracker's. * * @throws IOException When starting the cluster fails. @@ -527,7 +527,7 @@ public class HBaseTestingUtility { public void startMiniMapReduceCluster() throws IOException { startMiniMapReduceCluster(2); } - + /** * Starts a MiniMRCluster. * @@ -540,13 +540,13 @@ public class HBaseTestingUtility { Configuration c = getConfiguration(); System.setProperty("hadoop.log.dir", c.get("hadoop.log.dir")); c.set("mapred.output.dir", c.get("hadoop.tmp.dir")); - mrCluster = new MiniMRCluster(servers, + mrCluster = new MiniMRCluster(servers, FileSystem.get(c).getUri().toString(), 1); LOG.info("Mini mapreduce cluster started"); } - + /** - * Stops the previously started MiniMRCluster. + * Stops the previously started MiniMRCluster. */ public void shutdownMiniMapReduceCluster() { LOG.info("Stopping mini mapreduce cluster..."); @@ -608,13 +608,13 @@ public class HBaseTestingUtility { /** * Get the HBase cluster. - * + * * @return hbase cluster */ public MiniHBaseCluster getHBaseCluster() { return hbaseCluster; } - + /** * Returns a HBaseAdmin instance. * @@ -627,9 +627,9 @@ public class HBaseTestingUtility { } return hbaseAdmin; } - + /** - * Closes the named region. + * Closes the named region. * * @param regionName The region to close. * @throws IOException @@ -637,9 +637,9 @@ public class HBaseTestingUtility { public void closeRegion(String regionName) throws IOException { closeRegion(Bytes.toBytes(regionName)); } - + /** - * Closes the named region. + * Closes the named region. * * @param regionName The region to close. * @throws IOException @@ -648,9 +648,9 @@ public class HBaseTestingUtility { HBaseAdmin admin = getHBaseAdmin(); admin.closeRegion(regionName, (Object[]) null); } - + /** - * Closes the region containing the given row. + * Closes the region containing the given row. * * @param row The row to find the containing region. * @param table The table to find the region. @@ -661,7 +661,7 @@ public class HBaseTestingUtility { } /** - * Closes the region containing the given row. + * Closes the region containing the given row. * * @param row The row to find the containing region. * @param table The table to find the region. Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java Fri May 7 19:26:45 2010 @@ -42,14 +42,14 @@ import org.apache.hadoop.hbase.util.Byte *

*/ public class HFilePerformanceEvaluation { - + private static final int ROW_LENGTH = 10; private static final int ROW_COUNT = 1000000; private static final int RFILE_BLOCKSIZE = 8 * 1024; - + static final Log LOG = LogFactory.getLog(HFilePerformanceEvaluation.class.getName()); - + static byte [] format(final int i) { String v = Integer.toString(i); return Bytes.toBytes("0000000000".substring(v.length()) + v); @@ -110,9 +110,9 @@ public class HFilePerformanceEvaluation } } }); - + } - + protected void runBenchmark(RowOrientedBenchmark benchmark, int rowCount) throws Exception { LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + @@ -121,14 +121,14 @@ public class HFilePerformanceEvaluation LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + rowCount + " rows took " + elapsedTime + "ms."); } - + static abstract class RowOrientedBenchmark { - + protected final Configuration conf; protected final FileSystem fs; protected final Path mf; protected final int totalRows; - + public RowOrientedBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { this.conf = conf; @@ -136,21 +136,21 @@ public class HFilePerformanceEvaluation this.mf = mf; this.totalRows = totalRows; } - + void setUp() throws Exception { // do nothing } - + abstract void doRow(int i) throws Exception; - + protected int getReportingPeriod() { return this.totalRows / 10; } - + void tearDown() throws Exception { // do nothing } - + /** * Run benchmark * @return elapsed time. @@ -173,76 +173,76 @@ public class HFilePerformanceEvaluation } return elapsedTime; } - + } - + static class SequentialWriteBenchmark extends RowOrientedBenchmark { protected HFile.Writer writer; private Random random = new Random(); private byte[] bytes = new byte[ROW_LENGTH]; - + public SequentialWriteBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { super(conf, fs, mf, totalRows); } - + @Override void setUp() throws Exception { writer = new HFile.Writer(this.fs, this.mf, RFILE_BLOCKSIZE, (Compression.Algorithm) null, null); } - + @Override void doRow(int i) throws Exception { - writer.append(format(i), generateValue()); + writer.append(format(i), generateValue()); } - + private byte[] generateValue() { random.nextBytes(bytes); return bytes; } - + @Override protected int getReportingPeriod() { return this.totalRows; // don't report progress } - + @Override void tearDown() throws Exception { writer.close(); } - + } - + static abstract class ReadBenchmark extends RowOrientedBenchmark { - + protected HFile.Reader reader; - + public ReadBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { super(conf, fs, mf, totalRows); } - + @Override void setUp() throws Exception { reader = new HFile.Reader(this.fs, this.mf, null, false); this.reader.loadFileInfo(); } - + @Override void tearDown() throws Exception { reader.close(); } - + } static class SequentialReadBenchmark extends ReadBenchmark { private HFileScanner scanner; - + public SequentialReadBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { super(conf, fs, mf, totalRows); } - + @Override void setUp() throws Exception { super.setUp(); @@ -259,16 +259,16 @@ public class HFilePerformanceEvaluation PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); } } - + @Override protected int getReportingPeriod() { return this.totalRows; // don't report progress } - + } - + static class UniformRandomReadBenchmark extends ReadBenchmark { - + private Random random = new Random(); public UniformRandomReadBenchmark(Configuration conf, FileSystem fs, @@ -286,12 +286,12 @@ public class HFilePerformanceEvaluation ByteBuffer v = scanner.getValue(); PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); } - + private byte [] getRandomRow() { return format(random.nextInt(totalRows)); } } - + static class UniformRandomSmallScan extends ReadBenchmark { private Random random = new Random(); @@ -319,14 +319,14 @@ public class HFilePerformanceEvaluation PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); } } - + private byte [] getRandomRow() { return format(random.nextInt(totalRows)); } } - + static class GaussianRandomReadBenchmark extends ReadBenchmark { - + private RandomData randomData = new RandomDataImpl(); public GaussianRandomReadBenchmark(Configuration conf, FileSystem fs, @@ -353,11 +353,11 @@ public class HFilePerformanceEvaluation return format(r); } } - + /** * @param args - * @throws Exception - * @throws IOException + * @throws Exception + * @throws IOException */ public static void main(String[] args) throws Exception { new HFilePerformanceEvaluation().runBenchmarks(); Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MapFilePerformanceEvaluation.java Fri May 7 19:26:45 2010 @@ -43,7 +43,7 @@ public class MapFilePerformanceEvaluatio protected final HBaseConfiguration conf; private static final int ROW_LENGTH = 10; private static final int ROW_COUNT = 100000; - + static final Log LOG = LogFactory.getLog(MapFilePerformanceEvaluation.class.getName()); @@ -54,7 +54,7 @@ public class MapFilePerformanceEvaluatio super(); this.conf = c; } - + static ImmutableBytesWritable format(final int i, ImmutableBytesWritable w) { String v = Integer.toString(i); w.set(Bytes.toBytes("0000000000".substring(v.length()) + v)); @@ -69,7 +69,7 @@ public class MapFilePerformanceEvaluatio } runBenchmark(new SequentialWriteBenchmark(conf, fs, mf, ROW_COUNT), ROW_COUNT); - + PerformanceEvaluationCommons.concurrentReads(new Runnable() { public void run() { try { @@ -111,7 +111,7 @@ public class MapFilePerformanceEvaluatio } }); } - + protected void runBenchmark(RowOrientedBenchmark benchmark, int rowCount) throws Exception { LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + @@ -120,14 +120,14 @@ public class MapFilePerformanceEvaluatio LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + rowCount + " rows took " + elapsedTime + "ms."); } - + static abstract class RowOrientedBenchmark { - + protected final Configuration conf; protected final FileSystem fs; protected final Path mf; protected final int totalRows; - + public RowOrientedBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { this.conf = conf; @@ -135,21 +135,21 @@ public class MapFilePerformanceEvaluatio this.mf = mf; this.totalRows = totalRows; } - + void setUp() throws Exception { // do nothing } - + abstract void doRow(int i) throws Exception; - + protected int getReportingPeriod() { return this.totalRows / 10; } - + void tearDown() throws Exception { // do nothing } - + /** * Run benchmark * @return elapsed time. @@ -172,77 +172,77 @@ public class MapFilePerformanceEvaluatio } return elapsedTime; } - + } - + static class SequentialWriteBenchmark extends RowOrientedBenchmark { - + protected MapFile.Writer writer; private Random random = new Random(); private byte[] bytes = new byte[ROW_LENGTH]; private ImmutableBytesWritable key = new ImmutableBytesWritable(); private ImmutableBytesWritable value = new ImmutableBytesWritable(); - + public SequentialWriteBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { super(conf, fs, mf, totalRows); } - + @Override void setUp() throws Exception { writer = new MapFile.Writer(conf, fs, mf.toString(), ImmutableBytesWritable.class, ImmutableBytesWritable.class); } - + @Override void doRow(int i) throws Exception { value.set(generateValue()); - writer.append(format(i, key), value); + writer.append(format(i, key), value); } - + private byte[] generateValue() { random.nextBytes(bytes); return bytes; } - + @Override protected int getReportingPeriod() { return this.totalRows; // don't report progress } - + @Override void tearDown() throws Exception { writer.close(); } - + } - + static abstract class ReadBenchmark extends RowOrientedBenchmark { ImmutableBytesWritable key = new ImmutableBytesWritable(); ImmutableBytesWritable value = new ImmutableBytesWritable(); - + protected MapFile.Reader reader; - + public ReadBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { super(conf, fs, mf, totalRows); } - + @Override void setUp() throws Exception { reader = new MapFile.Reader(fs, mf.toString(), conf); } - + @Override void tearDown() throws Exception { reader.close(); } - + } static class SequentialReadBenchmark extends ReadBenchmark { ImmutableBytesWritable verify = new ImmutableBytesWritable(); - + public SequentialReadBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { super(conf, fs, mf, totalRows); @@ -255,16 +255,16 @@ public class MapFilePerformanceEvaluatio format(i, this.verify).get()); PerformanceEvaluationCommons.assertValueSize(ROW_LENGTH, value.getSize()); } - + @Override protected int getReportingPeriod() { return this.totalRows; // don't report progress } - + } - + static class UniformRandomReadBenchmark extends ReadBenchmark { - + private Random random = new Random(); public UniformRandomReadBenchmark(Configuration conf, FileSystem fs, @@ -278,13 +278,13 @@ public class MapFilePerformanceEvaluatio ImmutableBytesWritable r = (ImmutableBytesWritable)reader.get(k, value); PerformanceEvaluationCommons.assertValueSize(r.getSize(), ROW_LENGTH); } - + private ImmutableBytesWritable getRandomRow() { return format(random.nextInt(totalRows), key); } - + } - + static class UniformRandomSmallScan extends ReadBenchmark { private Random random = new Random(); @@ -308,7 +308,7 @@ public class MapFilePerformanceEvaluatio PerformanceEvaluationCommons.assertValueSize(this.value.getSize(), ROW_LENGTH); } } - + private ImmutableBytesWritable getRandomRow() { return format(random.nextInt(totalRows), key); } @@ -328,19 +328,19 @@ public class MapFilePerformanceEvaluatio ImmutableBytesWritable r = (ImmutableBytesWritable)reader.get(k, value); PerformanceEvaluationCommons.assertValueSize(r.getSize(), ROW_LENGTH); } - + private ImmutableBytesWritable getGaussianRandomRow() { int r = (int) randomData.nextGaussian((double)totalRows / 2.0, (double)totalRows / 10.0); return format(r, key); } - + } /** * @param args - * @throws Exception - * @throws IOException + * @throws Exception + * @throws IOException */ public static void main(String[] args) throws Exception { new MapFilePerformanceEvaluation(new HBaseConfiguration()). Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java Fri May 7 19:26:45 2010 @@ -49,12 +49,12 @@ public class MiniHBaseCluster implements public LocalHBaseCluster hbaseCluster; /** - * Start a MiniHBaseCluster. + * Start a MiniHBaseCluster. * @param conf Configuration to be used for cluster * @param numRegionServers initial number of region servers to start. * @throws IOException */ - public MiniHBaseCluster(Configuration conf, int numRegionServers) + public MiniHBaseCluster(Configuration conf, int numRegionServers) throws IOException { this.conf = conf; init(numRegionServers); @@ -239,7 +239,7 @@ public class MiniHBaseCluster implements /** * Shut down the mini HBase cluster - * @throws IOException + * @throws IOException */ public void shutdown() throws IOException { if (this.hbaseCluster != null) { Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MultiRegionTable.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MultiRegionTable.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MultiRegionTable.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/MultiRegionTable.java Fri May 7 19:26:45 2010 @@ -55,7 +55,7 @@ public class MultiRegionTable extends HB Bytes.toBytes("xxx"), Bytes.toBytes("yyy") }; - + protected final byte [] columnFamily; protected HTableDescriptor desc; @@ -68,13 +68,13 @@ public class MultiRegionTable extends HB public MultiRegionTable(int nServers, final String familyName) { super(nServers); - + this.columnFamily = Bytes.toBytes(familyName); // These are needed for the new and improved Map/Reduce framework System.setProperty("hadoop.log.dir", conf.get("hadoop.log.dir")); conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir")); } - + /** * Run after dfs is ready but before hbase cluster is started up. */ @@ -96,13 +96,13 @@ public class MultiRegionTable extends HB for(int i = 0; i < regions.length; i++) { HRegion.addRegionToMETA(meta, regions[i]); } - + closeRootAndMeta(); } catch (Exception e) { shutdownDfs(dfsCluster); throw e; } - } + } private HRegion createARegion(byte [] startKey, byte [] endKey) throws IOException { HRegion region = createNewHRegion(desc, startKey, endKey); @@ -110,7 +110,7 @@ public class MultiRegionTable extends HB closeRegionAndDeleteLog(region); return region; } - + private void closeRegionAndDeleteLog(HRegion region) throws IOException { region.close(); region.getLog().closeAndDelete(); Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java Fri May 7 19:26:45 2010 @@ -84,22 +84,22 @@ import org.apache.hadoop.util.LineReader * command-line which test to run and how many clients are participating in * this experiment. Run java PerformanceEvaluation --help to * obtain usage. - * + * *

This class sets up and runs the evaluation programs described in * Section 7, Performance Evaluation, of the Bigtable * paper, pages 8-10. - * + * *

If number of clients > 1, we start up a MapReduce job. Each map task * runs an individual client. Each client does about 1GB of data. */ public class PerformanceEvaluation implements HConstants { protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName()); - + private static final int ROW_LENGTH = 1000; private static final int ONE_GB = 1024 * 1024 * 1000; private static final int ROWS_PER_GB = ONE_GB / ROW_LENGTH; - + public static final byte [] TABLE_NAME = Bytes.toBytes("TestTable"); public static final byte [] FAMILY_NAME = Bytes.toBytes("info"); public static final byte [] QUALIFIER_NAME = Bytes.toBytes("data"); @@ -111,7 +111,7 @@ public class PerformanceEvaluation imple } protected Map commands = new TreeMap(); - + volatile Configuration conf; private boolean miniCluster = false; private boolean nomapred = false; @@ -127,9 +127,9 @@ public class PerformanceEvaluation imple public static final Pattern LINE_PATTERN = Pattern.compile("startRow=(\\d+),\\s+" + "perClientRunRows=(\\d+),\\s+" + - "totalRows=(\\d+),\\s+" + - "clients=(\\d+),\\s+" + - "flushCommits=(\\w+),\\s+" + + "totalRows=(\\d+),\\s+" + + "clients=(\\d+),\\s+" + + "flushCommits=(\\w+),\\s+" + "writeToWAL=(\\w+)"); /** @@ -141,8 +141,8 @@ public class PerformanceEvaluation imple ELAPSED_TIME, /** number of rows */ ROWS} - - + + /** * Constructor * @param c Configuration object @@ -174,13 +174,13 @@ public class PerformanceEvaluation imple "Run scan test using a filter to find a specific row based on it's value (make sure to use --rows=20)"); } - protected void addCommandDescriptor(Class cmdClass, + protected void addCommandDescriptor(Class cmdClass, String name, String description) { - CmdDescriptor cmdDescriptor = + CmdDescriptor cmdDescriptor = new CmdDescriptor(cmdClass, name, description); commands.put(name, cmdDescriptor); } - + /** * Implementations can have their status set. */ @@ -192,11 +192,11 @@ public class PerformanceEvaluation imple */ void setStatus(final String msg) throws IOException; } - + /** * This class works as the InputSplit of Performance Evaluation - * MapReduce InputFormat, and the Record Value of RecordReader. - * Each map task will only read one record from a PeInputSplit, + * MapReduce InputFormat, and the Record Value of RecordReader. + * Each map task will only read one record from a PeInputSplit, * the record value is the PeInputSplit itself. */ public static class PeInputSplit extends InputSplit implements Writable { @@ -206,7 +206,7 @@ public class PerformanceEvaluation imple private int clients = 0; private boolean flushCommits = false; private boolean writeToWAL = true; - + public PeInputSplit() { this.startRow = 0; this.rows = 0; @@ -215,7 +215,7 @@ public class PerformanceEvaluation imple this.flushCommits = false; this.writeToWAL = true; } - + public PeInputSplit(int startRow, int rows, int totalRows, int clients, boolean flushCommits, boolean writeToWAL) { this.startRow = startRow; @@ -225,13 +225,13 @@ public class PerformanceEvaluation imple this.flushCommits = flushCommits; this.writeToWAL = writeToWAL; } - + @Override public void readFields(DataInput in) throws IOException { this.startRow = in.readInt(); this.rows = in.readInt(); this.totalRows = in.readInt(); - this.clients = in.readInt(); + this.clients = in.readInt(); this.flushCommits = in.readBoolean(); this.writeToWAL = in.readBoolean(); } @@ -245,29 +245,29 @@ public class PerformanceEvaluation imple out.writeBoolean(flushCommits); out.writeBoolean(writeToWAL); } - + @Override public long getLength() throws IOException, InterruptedException { return 0; } - + @Override public String[] getLocations() throws IOException, InterruptedException { return new String[0]; } - + public int getStartRow() { return startRow; } - + public int getRows() { return rows; } - + public int getTotalRows() { return totalRows; } - + public int getClients() { return clients; } @@ -280,10 +280,10 @@ public class PerformanceEvaluation imple return writeToWAL; } } - + /** * InputFormat of Performance Evaluation MapReduce job. - * It extends from FileInputFormat, want to use it's methods such as setInputPaths(). + * It extends from FileInputFormat, want to use it's methods such as setInputPaths(). */ public static class PeInputFormat extends FileInputFormat { @@ -291,7 +291,7 @@ public class PerformanceEvaluation imple public List getSplits(JobContext job) throws IOException { // generate splits List splitList = new ArrayList(); - + for (FileStatus file: listStatus(job)) { Path path = file.getPath(); FileSystem fs = path.getFileSystem(job.getConfiguration()); @@ -313,7 +313,7 @@ public class PerformanceEvaluation imple boolean flushCommits = Boolean.parseBoolean(m.group(5)); boolean writeToWAL = Boolean.parseBoolean(m.group(6)); - LOG.debug("split["+ splitList.size() + "] " + + LOG.debug("split["+ splitList.size() + "] " + " startRow=" + startRow + " rows=" + rows + " totalRows=" + totalRows + @@ -322,60 +322,60 @@ public class PerformanceEvaluation imple " writeToWAL=" + writeToWAL); PeInputSplit newSplit = - new PeInputSplit(startRow, rows, totalRows, clients, + new PeInputSplit(startRow, rows, totalRows, clients, flushCommits, writeToWAL); splitList.add(newSplit); } } in.close(); } - + LOG.info("Total # of splits: " + splitList.size()); return splitList; } - + @Override public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) { return new PeRecordReader(); } - + public static class PeRecordReader extends RecordReader { private boolean readOver = false; private PeInputSplit split = null; private NullWritable key = null; private PeInputSplit value = null; - + @Override - public void initialize(InputSplit split, TaskAttemptContext context) + public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { this.readOver = false; this.split = (PeInputSplit)split; } - + @Override public boolean nextKeyValue() throws IOException, InterruptedException { if(readOver) { return false; } - + key = NullWritable.get(); value = (PeInputSplit)split; - + readOver = true; return true; } - + @Override public NullWritable getCurrentKey() throws IOException, InterruptedException { return key; } - + @Override public PeInputSplit getCurrentValue() throws IOException, InterruptedException { return value; } - + @Override public float getProgress() throws IOException, InterruptedException { if(readOver) { @@ -384,18 +384,18 @@ public class PerformanceEvaluation imple return 0.0f; } } - + @Override public void close() throws IOException { // do nothing } } } - + /** * MapReduce job that runs a performance evaluation client in each map task. */ - public static class EvaluationMapTask + public static class EvaluationMapTask extends Mapper { /** configuration parameter name that contains the command */ @@ -432,18 +432,18 @@ public class PerformanceEvaluation imple return clazz; } - protected void map(NullWritable key, PeInputSplit value, final Context context) + protected void map(NullWritable key, PeInputSplit value, final Context context) throws IOException, InterruptedException { - + Status status = new Status() { public void setStatus(String msg) { - context.setStatus(msg); + context.setStatus(msg); } }; - + // Evaluation task long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(), - value.getRows(), value.getTotalRows(), + value.getRows(), value.getTotalRows(), value.isFlushCommits(), value.isWriteToWAL(), status); // Collect how much time the thing took. Report as map output and @@ -454,7 +454,7 @@ public class PerformanceEvaluation imple context.progress(); } } - + /* * If table does not already exist, create. * @param c Client to use checking. @@ -490,7 +490,7 @@ public class PerformanceEvaluation imple doMapReduce(cmd); } } - + /* * Run all clients in this vm each to its own thread. * @param cmd Command to run. @@ -536,7 +536,7 @@ public class PerformanceEvaluation imple } } } - + /* * Run a mapreduce job. Run as many maps as asked-for clients. * Before we start up the job, write out an input file with instruction @@ -552,24 +552,24 @@ public class PerformanceEvaluation imple Job job = new Job(this.conf); job.setJarByClass(PerformanceEvaluation.class); job.setJobName("HBase Performance Evaluation"); - + job.setInputFormatClass(PeInputFormat.class); PeInputFormat.setInputPaths(job, inputDir); - + job.setOutputKeyClass(LongWritable.class); job.setOutputValueClass(LongWritable.class); - + job.setMapperClass(EvaluationMapTask.class); job.setReducerClass(LongSumReducer.class); - + job.setNumReduceTasks(1); - + job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, new Path(inputDir,"outputs")); - + job.waitForCompletion(true); } - + /* * Write input file of offsets-per-client for the mapreduce job. * @param c Configuration @@ -694,7 +694,7 @@ public class PerformanceEvaluation imple */ static abstract class Test { // Below is make it so when Tests are all running in the one - // jvm, that they each have a differently seeded Random. + // jvm, that they each have a differently seeded Random. private static final Random randomSeed = new Random(System.currentTimeMillis()); private static long nextRandomSeed() { @@ -729,16 +729,16 @@ public class PerformanceEvaluation imple this.flushCommits = options.isFlushCommits(); this.writeToWAL = options.isWriteToWAL(); } - + private String generateStatus(final int sr, final int i, final int lr) { return sr + "/" + i + "/" + lr; } - + protected int getReportingPeriod() { int period = this.perClientRunRows / 10; return period == 0? this.perClientRunRows: period; } - + void testSetup() throws IOException { this.admin = new HBaseAdmin(conf); this.table = new HTable(conf, tableName); @@ -752,7 +752,7 @@ public class PerformanceEvaluation imple this.table.flushCommits(); } } - + /* * Run test * @return Elapsed time. @@ -811,7 +811,7 @@ public class PerformanceEvaluation imple } s.close(); } - + @Override protected int getReportingPeriod() { int period = this.perClientRunRows / 100; @@ -924,12 +924,12 @@ public class PerformanceEvaluation imple } } - + static class RandomWriteTest extends Test { RandomWriteTest(Configuration conf, TestOptions options, Status status) { super(conf, options, status); } - + @Override void testRow(final int i) throws IOException { byte [] row = getRandomRow(this.rand, this.totalRows); @@ -940,19 +940,19 @@ public class PerformanceEvaluation imple table.put(put); } } - + static class ScanTest extends Test { private ResultScanner testScanner; ScanTest(Configuration conf, TestOptions options, Status status) { super(conf, options, status); } - + @Override void testSetup() throws IOException { super.testSetup(); } - + @Override void testTakedown() throws IOException { if (this.testScanner != null) { @@ -960,8 +960,8 @@ public class PerformanceEvaluation imple } super.testTakedown(); } - - + + @Override void testRow(final int i) throws IOException { if (this.testScanner == null) { @@ -973,12 +973,12 @@ public class PerformanceEvaluation imple } } - + static class SequentialReadTest extends Test { SequentialReadTest(Configuration conf, TestOptions options, Status status) { super(conf, options, status); } - + @Override void testRow(final int i) throws IOException { Get get = new Get(format(i)); @@ -987,12 +987,12 @@ public class PerformanceEvaluation imple } } - + static class SequentialWriteTest extends Test { SequentialWriteTest(Configuration conf, TestOptions options, Status status) { super(conf, options, status); } - + @Override void testRow(final int i) throws IOException { Put put = new Put(format(i)); @@ -1036,7 +1036,7 @@ public class PerformanceEvaluation imple return scan; } } - + /* * Format passed integer. * @param number @@ -1052,7 +1052,7 @@ public class PerformanceEvaluation imple } return b; } - + /* * This method takes some time and is done inline uploading data. For * example, doing the mapfile test, generation of the key and value @@ -1064,14 +1064,14 @@ public class PerformanceEvaluation imple r.nextBytes(b); return b; } - + static byte [] getRandomRow(final Random random, final int totalRows) { return format(random.nextInt(Integer.MAX_VALUE) % totalRows); } - + long runOneClient(final Class cmd, final int startRow, - final int perClientRunRows, final int totalRows, - boolean flushCommits, boolean writeToWAL, + final int perClientRunRows, final int totalRows, + boolean flushCommits, boolean writeToWAL, final Status status) throws IOException { status.setStatus("Start " + cmd + " at offset " + startRow + " for " + @@ -1099,7 +1099,7 @@ public class PerformanceEvaluation imple "ms at offset " + startRow + " for " + perClientRunRows + " rows"); return totalElapsedTime; } - + private void runNIsOne(final Class cmd) { Status status = new Status() { public void setStatus(String msg) throws IOException { @@ -1115,7 +1115,7 @@ public class PerformanceEvaluation imple status); } catch (Exception e) { LOG.error("Failed", e); - } + } } private void runTest(final Class cmd) throws IOException, @@ -1127,7 +1127,7 @@ public class PerformanceEvaluation imple dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null); zooKeeperCluster = new MiniZooKeeperCluster(); int zooKeeperPort = zooKeeperCluster.startup(new File(System.getProperty("java.io.tmpdir"))); - + // mangle the conf so that the fs parameter points to the minidfs we // just started up FileSystem fs = dfsCluster.getFileSystem(); @@ -1139,14 +1139,14 @@ public class PerformanceEvaluation imple FSUtils.setVersion(fs, parentdir); hbaseMiniCluster = new MiniHBaseCluster(this.conf, N); } - + try { if (N == 1) { // If there is only one client and one HRegionServer, we assume nothing // has been set up at all. runNIsOne(cmd); } else { - // Else, run + // Else, run runNIsMoreThanOne(cmd); } } finally { @@ -1157,11 +1157,11 @@ public class PerformanceEvaluation imple } } } - + protected void printUsage() { printUsage(null); } - + protected void printUsage(final String message) { if (message != null && message.length() > 0) { System.err.println(message); @@ -1203,16 +1203,16 @@ public class PerformanceEvaluation imple // Set total number of rows to write. this.R = this.R * N; } - + public int doCommandLine(final String[] args) { // Process command-line args. TODO: Better cmd-line processing - // (but hopefully something not as painful as cli options). + // (but hopefully something not as painful as cli options). int errCode = -1; if (args.length < 1) { printUsage(); return errCode; } - + try { for (int i = 0; i < args.length; i++) { String cmd = args[i]; @@ -1221,19 +1221,19 @@ public class PerformanceEvaluation imple errCode = 0; break; } - + final String miniClusterArgKey = "--miniCluster"; if (cmd.startsWith(miniClusterArgKey)) { this.miniCluster = true; continue; } - + final String nmr = "--nomapred"; if (cmd.startsWith(nmr)) { this.nomapred = true; continue; } - + final String rows = "--rows="; if (cmd.startsWith(rows)) { this.R = Integer.parseInt(cmd.substring(rows.length())); @@ -1259,14 +1259,14 @@ public class PerformanceEvaluation imple errCode = 0; break; } - + printUsage(); break; } } catch (Exception e) { e.printStackTrace(); } - + return errCode; } Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java Fri May 7 19:26:45 2010 @@ -33,13 +33,13 @@ import org.apache.commons.logging.LogFac public class PerformanceEvaluationCommons { static final Log LOG = LogFactory.getLog(PerformanceEvaluationCommons.class.getName()); - + public static void assertValueSize(final int expectedSize, final int got) { if (got != expectedSize) { throw new AssertionError("Expected " + expectedSize + " but got " + got); } } - + public static void assertKey(final byte [] expected, final ByteBuffer got) { byte [] b = new byte[got.limit()]; got.get(b, 0, got.limit()); @@ -53,7 +53,7 @@ public class PerformanceEvaluationCommon " but got " + org.apache.hadoop.hbase.util.Bytes.toString(got)); } } - + public static void concurrentReads(final Runnable r) { final int count = 1; long now = System.currentTimeMillis(); Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestCompare.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestCompare.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestCompare.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestCompare.java Fri May 7 19:26:45 2010 @@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.util.Byte * Test comparing HBase objects. */ public class TestCompare extends TestCase { - + /** * Sort of HRegionInfo. */ Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestHMsg.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestHMsg.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestHMsg.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestHMsg.java Fri May 7 19:26:45 2010 @@ -54,7 +54,7 @@ public class TestHMsg extends TestCase { new HRegionInfo(new HTableDescriptor(Bytes.toBytes("test")), b, b)); assertNotSame(-1, msgs.indexOf(hmsg)); } - + public void testSerialization() throws IOException { // Check out new HMsg that carries two daughter split regions. byte [] abytes = Bytes.toBytes("a"); Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestInfoServers.java Fri May 7 19:26:45 2010 @@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.client.HT */ public class TestInfoServers extends HBaseClusterTestCase { static final Log LOG = LogFactory.getLog(TestInfoServers.class); - + @Override protected void preHBaseClusterSetup() { // Bring up info servers on 'odd' port numbers in case the test is not @@ -41,7 +41,7 @@ public class TestInfoServers extends HBa conf.setInt("hbase.master.info.port", 60011); conf.setInt("hbase.regionserver.info.port", 60031); } - + /** * @throws Exception */ @@ -56,7 +56,7 @@ public class TestInfoServers extends HBa assertHasExpectedContent(new URL("http://localhost:" + port + "/index.html"), "regionserver"); } - + private void assertHasExpectedContent(final URL u, final String expected) throws IOException { LOG.info("Testing " + u.toString() + " has " + expected); Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java Fri May 7 19:26:45 2010 @@ -40,7 +40,7 @@ public class TestKeyValue extends TestCa byte [] qualifier1 = Bytes.toBytes("def"); byte [] family2 = Bytes.toBytes("abcd"); byte [] qualifier2 = Bytes.toBytes("ef"); - + KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, Type.Put, a); assertFalse(aaa.matchingColumn(family2, qualifier2)); assertTrue(aaa.matchingColumn(family1, qualifier1)); @@ -62,7 +62,7 @@ public class TestKeyValue extends TestCa check(Bytes.toBytes(getName()), Bytes.toBytes(getName()), null, 1, null); check(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes(getName()), null, 1, null); } - + private void check(final byte [] row, final byte [] family, byte [] qualifier, final long timestamp, final byte [] value) { KeyValue kv = new KeyValue(row, family, qualifier, timestamp, value); @@ -71,7 +71,7 @@ public class TestKeyValue extends TestCa // Call toString to make sure it works. LOG.info(kv.toString()); } - + public void testPlainCompare() throws Exception { final byte [] a = Bytes.toBytes("aaa"); final byte [] b = Bytes.toBytes("bbb"); @@ -118,11 +118,11 @@ public class TestKeyValue extends TestCa KVComparator c = new KeyValue.RootComparator(); assertTrue(c.compare(b, a) < 0); KeyValue aa = new KeyValue(Bytes.toBytes(".META.,,1"), now); - KeyValue bb = new KeyValue(Bytes.toBytes(".META.,,1"), + KeyValue bb = new KeyValue(Bytes.toBytes(".META.,,1"), Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1235943454602L, (byte[])null); assertTrue(c.compare(aa, bb) < 0); - + // Meta compares KeyValue aaa = new KeyValue( Bytes.toBytes("TestScanMultipleVersions,row_0500,1236020145502"), now); @@ -130,12 +130,12 @@ public class TestKeyValue extends TestCa Bytes.toBytes("TestScanMultipleVersions,,99999999999999"), now); c = new KeyValue.MetaComparator(); assertTrue(c.compare(bbb, aaa) < 0); - + KeyValue aaaa = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,,1236023996656"), Bytes.toBytes("info"), Bytes.toBytes("regioninfo"), 1236024396271L, (byte[])null); assertTrue(c.compare(aaaa, bbb) < 0); - + KeyValue x = new KeyValue(Bytes.toBytes("TestScanMultipleVersions,row_0500,1236034574162"), Bytes.toBytes("info"), Bytes.toBytes(""), 9223372036854775807L, (byte[])null); @@ -152,7 +152,7 @@ public class TestKeyValue extends TestCa /** * Tests cases where rows keys have characters below the ','. * See HBASE-832 - * @throws IOException + * @throws IOException */ public void testKeyValueBorderCases() throws IOException { // % sorts before , so if we don't do special comparator, rowB would @@ -163,15 +163,15 @@ public class TestKeyValue extends TestCa Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0); - rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"), + rowA = new KeyValue(Bytes.toBytes("testtable,,1234"), Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); - rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"), + rowB = new KeyValue(Bytes.toBytes("testtable,$www.hbase.org/,99999"), Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); assertTrue(KeyValue.META_COMPARATOR.compare(rowA, rowB) < 0); - rowA = new KeyValue(Bytes.toBytes(".META.,testtable,www.hbase.org/,1234,4321"), + rowA = new KeyValue(Bytes.toBytes(".META.,testtable,www.hbase.org/,1234,4321"), Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); - rowB = new KeyValue(Bytes.toBytes(".META.,testtable,www.hbase.org/%20,99999,99999"), + rowB = new KeyValue(Bytes.toBytes(".META.,testtable,www.hbase.org/%20,99999,99999"), Bytes.toBytes("fam"), Bytes.toBytes(""), Long.MAX_VALUE, (byte[])null); assertTrue(KeyValue.ROOT_COMPARATOR.compare(rowA, rowB) < 0); } @@ -275,6 +275,6 @@ public class TestKeyValue extends TestCa // Test multiple KeyValues in a single blob. // TODO actually write this test! - + } } Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestMergeMeta.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestMergeMeta.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestMergeMeta.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestMergeMeta.java Fri May 7 19:26:45 2010 @@ -24,7 +24,7 @@ import java.io.IOException; /** Tests region merging */ public class TestMergeMeta extends AbstractMergeTestBase { - /** constructor + /** constructor * @throws Exception */ public TestMergeMeta() throws Exception { @@ -32,7 +32,7 @@ public class TestMergeMeta extends Abstr conf.setLong("hbase.client.pause", 1 * 1000); conf.setInt("hbase.client.retries.number", 2); } - + /** * test case * @throws IOException Modified: hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=942186&r1=942185&r2=942186&view=diff ============================================================================== --- hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java (original) +++ hadoop/hbase/trunk/core/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java Fri May 7 19:26:45 2010 @@ -42,11 +42,11 @@ public class TestRegionRebalancing exten HTable table; HTableDescriptor desc; - + final byte[] FIVE_HUNDRED_KBYTES; - + final byte [] FAMILY_NAME = Bytes.toBytes("col"); - + /** constructor */ public TestRegionRebalancing() { super(1); @@ -54,11 +54,11 @@ public class TestRegionRebalancing exten for (int i = 0; i < 500 * 1024; i++) { FIVE_HUNDRED_KBYTES[i] = 'x'; } - + desc = new HTableDescriptor("test"); desc.addFamily(new HColumnDescriptor(FAMILY_NAME)); } - + /** * Before the hbase cluster starts up, create some dummy regions. */ @@ -72,32 +72,32 @@ public class TestRegionRebalancing exten } startKeys.add(null); LOG.info(startKeys.size() + " start keys generated"); - + List regions = new ArrayList(); for (int i = 0; i < 20; i++) { regions.add(createAregion(startKeys.get(i), startKeys.get(i+1))); } - + // Now create the root and meta regions and insert the data regions // created above into the meta - + createRootAndMetaRegions(); for (HRegion region : regions) { HRegion.addRegionToMETA(meta, region); } closeRootAndMeta(); } - + /** * For HBASE-71. Try a few different configurations of starting and stopping * region servers to see if the assignment or regions is pretty balanced. - * @throws IOException + * @throws IOException */ public void testRebalancing() throws IOException { table = new HTable(conf, "test"); - assertEquals("Test table should have 20 regions", + assertEquals("Test table should have 20 regions", 20, table.getStartKeys().length); - + // verify that the region assignments are balanced to start out assertRegionsAreBalanced(); @@ -105,32 +105,32 @@ public class TestRegionRebalancing exten // add a region server - total of 2 cluster.startRegionServer(); assertRegionsAreBalanced(); - + // add a region server - total of 3 - LOG.debug("Adding 3rd region server."); + LOG.debug("Adding 3rd region server."); cluster.startRegionServer(); assertRegionsAreBalanced(); - + // kill a region server - total of 2 LOG.debug("Killing the 3rd region server."); cluster.stopRegionServer(2, false); cluster.waitOnRegionServer(2); assertRegionsAreBalanced(); - + // start two more region servers - total of 4 LOG.debug("Adding 3rd region server"); cluster.startRegionServer(); - LOG.debug("Adding 4th region server"); - cluster.startRegionServer(); + LOG.debug("Adding 4th region server"); + cluster.startRegionServer(); assertRegionsAreBalanced(); for (int i = 0; i < 6; i++){ - LOG.debug("Adding " + (i + 5) + "th region server"); + LOG.debug("Adding " + (i + 5) + "th region server"); cluster.startRegionServer(); } assertRegionsAreBalanced(); } - + /** figure out how many regions are currently being served. */ private int getRegionCount() { int total = 0; @@ -139,7 +139,7 @@ public class TestRegionRebalancing exten } return total; } - + /** * Determine if regions are balanced. Figure out the total, divide by the * number of online servers, then test if each server is +/- 1 of average @@ -160,39 +160,39 @@ public class TestRegionRebalancing exten double avg = cluster.getMaster().getAverageLoad(); int avgLoadPlusSlop = (int)Math.ceil(avg * (1 + slop)); int avgLoadMinusSlop = (int)Math.floor(avg * (1 - slop)) - 1; - LOG.debug("There are " + servers.size() + " servers and " + regionCount + LOG.debug("There are " + servers.size() + " servers and " + regionCount + " regions. Load Average: " + avg + " low border: " + avgLoadMinusSlop + ", up border: " + avgLoadPlusSlop + "; attempt: " + i); for (HRegionServer server : servers) { int serverLoad = server.getOnlineRegions().size(); LOG.debug(server.hashCode() + " Avg: " + avg + " actual: " + serverLoad); - if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop + if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop && serverLoad >= avgLoadMinusSlop)) { LOG.debug(server.hashCode() + " Isn't balanced!!! Avg: " + avg + " actual: " + serverLoad + " slop: " + slop); success = false; } } - + if (!success) { - // one or more servers are not balanced. sleep a little to give it a + // one or more servers are not balanced. sleep a little to give it a // chance to catch up. then, go back to the retry loop. try { Thread.sleep(10000); } catch (InterruptedException e) {} - + continue; } - + // if we get here, all servers were balanced, so we should just return. return; } - // if we get here, we tried 5 times and never got to short circuit out of + // if we get here, we tried 5 times and never got to short circuit out of // the retry loop, so this is a failure. fail("After 5 attempts, region assignments were not balanced."); } - + private List getOnlineRegionServers() { List list = new ArrayList(); for (JVMClusterUtil.RegionServerThread rst : cluster.getRegionServerThreads()) { @@ -204,7 +204,7 @@ public class TestRegionRebalancing exten } /** - * Wait until all the regions are assigned. + * Wait until all the regions are assigned. */ private void waitForAllRegionsAssigned() { while (getRegionCount() < 22) { @@ -218,9 +218,9 @@ public class TestRegionRebalancing exten /** * create a region with the specified start and end key and exactly one row - * inside. + * inside. */ - private HRegion createAregion(byte [] startKey, byte [] endKey) + private HRegion createAregion(byte [] startKey, byte [] endKey) throws IOException { HRegion region = createNewHRegion(desc, startKey, endKey); byte [] keyToWrite = startKey == null ? Bytes.toBytes("row_000") : startKey;