Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 07EC2F395 for ; Tue, 26 Mar 2013 18:19:05 +0000 (UTC) Received: (qmail 23543 invoked by uid 500); 26 Mar 2013 18:19:04 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 23457 invoked by uid 500); 26 Mar 2013 18:19:04 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 23447 invoked by uid 99); 26 Mar 2013 18:19:04 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Mar 2013 18:19:04 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Mar 2013 18:19:00 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 0B36B2388847; Tue, 26 Mar 2013 18:18:39 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1461253 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java Date: Tue, 26 Mar 2013 18:18:38 -0000 To: commits@hbase.apache.org From: liyin@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130326181839.0B36B2388847@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: liyin Date: Tue Mar 26 18:18:38 2013 New Revision: 1461253 URL: http://svn.apache.org/r1461253 Log: [HBASE-8194] Fixing the non deterministic failure of TestHFileOutputFormat. Author: manukranthk Summary: The test creates regions in the meta region and waits for the master to allot them to regionservers. When the favored nodes were set, the timout was not completely reliable to say that they will be set. Instead setting the favored nodes while creating the regions directly in the starting would ensure that the master creates the regions assigning selected favored nodes. Test Plan: Unit Tests Reviewers: liyintang, rshroff, adela Reviewed By: rshroff CC: hbase-eng@ Differential Revision: https://phabricator.fb.com/D728493 Task ID: 2138207 Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java?rev=1461253&r1=1461252&r2=1461253&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java Tue Mar 26 18:18:38 2013 @@ -88,7 +88,7 @@ public class HFileOutputFormat extends F "hbase.hfileoutputformat.families.bloomfilter.typePerCF"; static final String ENCODING_TYPE_PER_CF_KEY = - "hbase.hfileoutputformat.families.encoding.typePerCF"; + "hbase.hfileoutputformat.families.encoding.typePerCF"; static final String TABLE_NAME = "hbase.hfileoutputformat.tablename"; static final String UTF8 = "UTF-8"; @@ -110,7 +110,7 @@ public class HFileOutputFormat extends F Compression.Algorithm.NONE.getName()); HTable tempTable = null; if (conf.get(TABLE_NAME) != null) { - tempTable = new HTable(conf, conf.get(TABLE_NAME)); + tempTable = new HTable(conf, conf.get(TABLE_NAME)); } final HTable table = tempTable; @@ -120,10 +120,10 @@ public class HFileOutputFormat extends F final Map bloomTypeMap = createFamilyBloomTypeMap(conf); final Map encoderTypeMap = - createFamilyDeltaEncodingMap(conf); + createFamilyDeltaEncodingMap(conf); final Pair startKeysAndFavoredNodes = - (table == null ? null : table.getStartKeysAndFavoredNodes()); + (table == null ? null : table.getStartKeysAndFavoredNodes()); return new RecordWriter() { // Map of families to writers and how much has been output on the writer. @@ -180,21 +180,21 @@ public class HFileOutputFormat extends F } private byte[] getFavoredNodesForKey(byte[] rowKey) { - if (startKeysAndFavoredNodes == null) { - return HConstants.EMPTY_BYTE_ARRAY; - } - byte[][] startKeys = startKeysAndFavoredNodes.getFirst(); - byte[][] favoredNodes = startKeysAndFavoredNodes.getSecond(); - if (startKeys == null || favoredNodes == null) - return HConstants.EMPTY_BYTE_ARRAY; - ConcurrentSkipListMap startKeysToFavoredNodes = - new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); - for (int i=0; i startKeysToFavoredNodes = + new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); + for (int i=0; i(startKeys, null)); + } + + public int createMultiRegionsWithFavoredNodes(final Configuration c, final HTable table, + final byte[] columnFamily, Pair startKeysAndFavNodes) + throws IOException { + byte[][] startKeys = startKeysAndFavNodes.getFirst(); + byte[][] favNodes = startKeysAndFavNodes.getSecond(); Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR); HTable meta = new HTable(c, HConstants.META_TABLE_NAME); HTableDescriptor htd = table.getTableDescriptor(); @@ -783,6 +797,10 @@ public class HBaseTestingUtility { Put put = new Put(hri.getRegionName()); put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(hri)); + if (favNodes != null) { + put.add(HConstants.CATALOG_FAMILY, HConstants.FAVOREDNODES_QUALIFIER, + favNodes[i]); + } meta.put(put); LOG.info("createMultiRegions: inserted " + hri.toString()); count++; @@ -1232,7 +1250,7 @@ TOP_LOOP: Threads.sleepWithoutInterrupt(2000); continue; } catch (PreemptiveFastFailException ex) { - // Be more patient + // Be more patient Threads.sleepWithoutInterrupt(2000); continue; } Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java?rev=1461253&r1=1461252&r2=1461253&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java Tue Mar 26 18:18:38 2013 @@ -76,6 +76,7 @@ import org.apache.hadoop.hbase.regionser import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -876,9 +877,9 @@ public class TestHFileOutputFormat { Map metadataMap = reader.loadFileInfo(); assertTrue("timeRange is not set", - metadataMap.get(StoreFile.TIMERANGE_KEY) != null); + metadataMap.get(StoreFile.TIMERANGE_KEY) != null); assertEquals("Incorrect bloom type used for column family " + - familyStr + "(reader: " + reader + ")", + familyStr + "(reader: " + reader + ")", configuredBloomFilter.get(familyStr), reader.getBloomFilterType()); break; @@ -896,24 +897,24 @@ public class TestHFileOutputFormat { } private void setupColumnFamiliesEncodingType(HTable table, - Map familyToEncoding) throws IOException { - HTableDescriptor mockTableDesc = new HTableDescriptor(); - for (Entry entry : familyToEncoding.entrySet()) { - mockTableDesc.addFamily( - new HColumnDescriptor(entry.getKey().getBytes(), - 1, - Compression.Algorithm.NONE.toString(), - true, - entry.getValue().toString(), - false, - false, - HColumnDescriptor.DEFAULT_BLOCKSIZE, - 0, - BloomType.NONE.toString(), - HColumnDescriptor.DEFAULT_REPLICATION_SCOPE, - HColumnDescriptor.DEFAULT_BLOOMFILTER_ERROR_RATE)); - } - Mockito.doReturn(mockTableDesc).when(table).getTableDescriptor(); + Map familyToEncoding) throws IOException { + HTableDescriptor mockTableDesc = new HTableDescriptor(); + for (Entry entry : familyToEncoding.entrySet()) { + mockTableDesc.addFamily( + new HColumnDescriptor(entry.getKey().getBytes(), + 1, + Compression.Algorithm.NONE.toString(), + true, + entry.getValue().toString(), + false, + false, + HColumnDescriptor.DEFAULT_BLOCKSIZE, + 0, + BloomType.NONE.toString(), + HColumnDescriptor.DEFAULT_REPLICATION_SCOPE, + HColumnDescriptor.DEFAULT_BLOOMFILTER_ERROR_RATE)); + } + Mockito.doReturn(mockTableDesc).when(table).getTableDescriptor(); } /** @@ -936,8 +937,8 @@ public class TestHFileOutputFormat { int familyIndex = 0; for (byte[] family : FAMILIES) { - configuredEncoding.put(Bytes.toString(family), - EncodingTypeValues[familyIndex++ % EncodingTypeValues.length]); + configuredEncoding.put(Bytes.toString(family), + EncodingTypeValues[familyIndex++ % EncodingTypeValues.length]); } setupColumnFamiliesEncodingType(table, configuredEncoding); @@ -984,7 +985,7 @@ public class TestHFileOutputFormat { assertTrue("timeRange is not set", metadataMap.get(StoreFile.TIMERANGE_KEY) != null); assertEquals("Incorrect Encoding Type used for column family " - + familyStr + "(reader: " + reader + ")", + + familyStr + "(reader: " + reader + ")", configuredEncoding.get(familyStr), reader.getHFileReader().getEncodingOnDisk()); break; @@ -1000,15 +1001,15 @@ public class TestHFileOutputFormat { dir.getFileSystem(conf).delete(dir, true); } } - + @Test public void testFavoredNodes() throws Exception { - Random rand = new Random(); - for (int i=0; i<3; i++) { - int tmp = (int)'b'; - byte c = (byte)(tmp + (Math.abs(rand.nextInt()))%24); - testFavoredNodesPerChar(c); - } + Random rand = new Random(); + for (int i=0; i<3; i++) { + int tmp = (int)'b'; + byte c = (byte)(tmp + (Math.abs(rand.nextInt()))%24); + testFavoredNodesPerChar(c); + } } private static final int FAVORED_NODES_NUM = 3; private static final int REGION_SERVERS = 10; @@ -1016,65 +1017,51 @@ public class TestHFileOutputFormat { * Testing FavoredNodes support for HFileOutputFormat */ public void testFavoredNodesPerChar(byte c) throws Exception{ - util.startMiniCluster(REGION_SERVERS); - Configuration conf = new Configuration(this.util.getConfiguration()); + util.startMiniCluster(REGION_SERVERS); + Configuration conf = new Configuration(this.util.getConfiguration()); RecordWriter writer = null; TaskAttemptContext context = null; Path dir = util.getTestDir("TestFavoredNodes"); byte[] familyName = Bytes.toBytes("family"); byte[] tableName = Bytes.toBytes("TestFavoredNodes"); - HTable table = util.createTable(tableName, familyName); - int countOfRegions = util.createMultiRegions(table, familyName); - util.waitUntilAllRegionsAssigned(countOfRegions); - - InetSocketAddress[] nodes = new InetSocketAddress[REGION_SERVERS]; - List datanodes = util.getDFSCluster().getDataNodes(); - for (int i = 0; i < REGION_SERVERS; i++) { - nodes[i] = datanodes.get(i).getSelfAddr(); - } - + byte[][] startKeys = util.getTmpKeys(); String[] nodeNames = new String[REGION_SERVERS]; + List dataNodes = util.getDFSCluster().getDataNodes(); for (int i = 0; i < REGION_SERVERS; i++) { - nodeNames[i] = nodes[i].getAddress().getHostAddress() + ":" + - nodes[i].getPort(); + DataNode node = dataNodes.get(i); + nodeNames[i] = + node.getSelfAddr().getAddress().getHostAddress() + ":" + + node.getSelfAddr().getPort(); } - - List puts = new ArrayList(); int testIndex = 0; - List regions = util.getHBaseCluster().getRegions(tableName); - for (int i = 0; i < regions.size(); i++) { - List favoredNodes = new ArrayList(FAVORED_NODES_NUM); - HRegion region = regions.get(i); - if (Bytes.BYTES_COMPARATOR.compare(region.getStartKey(), HConstants.EMPTY_BYTE_ARRAY) != 0) { - if (region.getStartKey()[0] == c) { - testIndex = i; - } - } - for (int j = 0; j < FAVORED_NODES_NUM; j++) { - favoredNodes.add(new HServerAddress(nodeNames[(i + j) % REGION_SERVERS])); + byte[][] favNodes = new byte[startKeys.length][]; + for (int i = 0; i < startKeys.length; i++) { + List favoredNodes = + new ArrayList(FAVORED_NODES_NUM); + if (i>0 && startKeys[i][0] == c) { + testIndex = i; + } + for (int j = 0; j < FAVORED_NODES_NUM; j++) { + favoredNodes.add(new HServerAddress(nodeNames[(i + j) % + REGION_SERVERS])); } - String favoredNodesString = RegionPlacement.getFavoredNodes(favoredNodes); - Put put = new Put(region.getRegionName()); - put.add(HConstants.CATALOG_FAMILY, HConstants.FAVOREDNODES_QUALIFIER, - favoredNodesString.getBytes()); - puts.add(put); + String favoredNodesString = + RegionPlacement.getFavoredNodes(favoredNodes); + byte[] favoredNodesBytes = Bytes.toBytes(favoredNodesString); + favNodes[i] = favoredNodesBytes; } + HTable table = util.createTable(tableName, familyName); + int countOfRegions = util.createMultiRegionsWithFavoredNodes(conf, table, familyName, + new Pair(startKeys, favNodes)); + util.waitUntilAllRegionsAssigned(countOfRegions); - // Write the region assignments to the meta table. - HTable metaTable = new HTable(conf, HConstants.META_TABLE_NAME); - metaTable.put(puts); - LOG.info("Updated the META with the new assignment plan"); - - // Allowing the Master thread to rescan and clean the empty meta rows - int sleepTime = conf.getInt("hbase.master.meta.thread.rescanfrequency", 100*1000); - Thread.sleep(sleepTime); try { Job job = new Job(conf, "testLocalMRIncrementalLoad"); setupRandomGeneratorMapper(job); HFileOutputFormat.configureIncrementalLoad(job, table); FileOutputFormat.setOutputPath(job, dir); context = new TaskAttemptContext(job.getConfiguration(), - new TaskAttemptID()); + new TaskAttemptID()); HFileOutputFormat hof = new HFileOutputFormat(); writer = hof.getRecordWriter(context); @@ -1110,22 +1097,22 @@ public class TestHFileOutputFormat { } } } finally { - dir.getFileSystem(conf).delete(dir, true); - util.shutdownMiniCluster(); + dir.getFileSystem(conf).delete(dir, true); + util.shutdownMiniCluster(); } } private void writeKVs(RecordWriter writer, - byte[] family, byte keyByte) throws IOException, InterruptedException{ - byte[] k = new byte[3]; - int b1 = (int)keyByte; - Random rand = new Random(); - int tmp = rand.nextInt(); - int b2 = Math.min(b1 + Math.abs(tmp)%26, (int)'z'); - tmp = rand.nextInt(); - int b3 = Math.min(b1 + Math.abs(tmp)%26, (int)'z'); - - for (byte byte2 = (byte)b2; byte2 <= 'z'; byte2++) { + byte[] family, byte keyByte) throws IOException, InterruptedException{ + byte[] k = new byte[3]; + int b1 = (int)keyByte; + Random rand = new Random(); + int tmp = rand.nextInt(); + int b2 = Math.min(b1 + Math.abs(tmp)%26, (int)'z'); + tmp = rand.nextInt(); + int b3 = Math.min(b1 + Math.abs(tmp)%26, (int)'z'); + + for (byte byte2 = (byte)b2; byte2 <= 'z'; byte2++) { for (byte byte3 = (byte)b3; byte3 <= 'z'; byte3++) { k[0] = (byte)b1; k[1] = byte2;