Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9A66489B8 for ; Fri, 9 Sep 2011 05:26:08 +0000 (UTC) Received: (qmail 99728 invoked by uid 500); 9 Sep 2011 05:26:06 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 98516 invoked by uid 500); 9 Sep 2011 05:25:51 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 98495 invoked by uid 99); 9 Sep 2011 05:25:49 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 09 Sep 2011 05:25:49 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 09 Sep 2011 05:25:46 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id EC09123889B3 for ; Fri, 9 Sep 2011 05:25:25 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1167006 - in /hbase/trunk: ./ src/main/java/org/apache/hadoop/hbase/catalog/ src/main/java/org/apache/hadoop/hbase/client/ src/main/java/org/apache/hadoop/hbase/master/ src/main/java/org/apache/hadoop/hbase/util/ Date: Fri, 09 Sep 2011 05:25:25 -0000 To: commits@hbase.apache.org From: tedyu@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110909052525.EC09123889B3@eris.apache.org> Author: tedyu Date: Fri Sep 9 05:25:24 2011 New Revision: 1167006 URL: http://svn.apache.org/viewvc?rev=1167006&view=rev Log: HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer) Modified: hbase/trunk/CHANGES.txt hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Modified: hbase/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1167006&r1=1167005&r2=1167006&view=diff ============================================================================== --- hbase/trunk/CHANGES.txt (original) +++ hbase/trunk/CHANGES.txt Fri Sep 9 05:25:24 2011 @@ -255,7 +255,7 @@ Release 0.91.0 - Unreleased TestMultiColumnScanner that caught it (Mikhail Bautin) HBASE-4007 distributed log splitting can get indefinitely stuck (Prakash Khemani) - + HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer) IMPROVEMENTS Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1167006&r1=1167005&r2=1167006&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java Fri Sep 9 05:25:24 2011 @@ -231,27 +231,23 @@ public class MetaEditor { } /** - * Updates the region information for the specified region in META. + * Update the metamigrated flag in -ROOT-. * @param catalogTracker - * @param regionInfo region to be updated in META * @throws IOException */ - public static void updateRegionInfo(CatalogTracker catalogTracker, - HRegionInfo regionInfo) - throws IOException { - Put put = new Put(regionInfo.getRegionName()); - addRegionInfo(put, regionInfo); - catalogTracker.waitForMetaServerConnectionDefault().put( - CatalogTracker.META_REGION, put); - LOG.info("Updated region " + regionInfo.getRegionNameAsString() + " in META"); - } - - public static void updateRootWithMetaMigrationStatus(CatalogTracker catalogTracker) throws IOException { + public static void updateRootWithMetaMigrationStatus( + CatalogTracker catalogTracker) throws IOException { updateRootWithMetaMigrationStatus(catalogTracker, true); } - public static void updateRootWithMetaMigrationStatus(CatalogTracker catalogTracker, - boolean metaUpdated) + /** + * Update the metamigrated flag in -ROOT-. + * @param catalogTracker + * @param metaUpdated + * @throws IOException + */ + public static void updateRootWithMetaMigrationStatus( + CatalogTracker catalogTracker, boolean metaUpdated) throws IOException { Put put = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName()); addMetaUpdateStatus(put, metaUpdated); @@ -260,6 +256,12 @@ public class MetaEditor { LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated); } + /** + * Update legacy META rows, removing HTD from HRI. + * @param masterServices + * @return + * @throws IOException + */ public static List updateMetaWithNewRegionInfo( final MasterServices masterServices) throws IOException { @@ -271,16 +273,11 @@ public class MetaEditor { HRegionInfo090x hrfm = getHRegionInfoForMigration(r); if (hrfm == null) return true; htds.add(hrfm.getTableDesc()); - masterServices.getMasterFileSystem().createTableDescriptor(hrfm.getTableDesc()); - HRegionInfo regionInfo = new HRegionInfo(hrfm); - LOG.debug(" MetaEditor.updatemeta RegionInfo = " + regionInfo.toString() - + " old HRI = " + hrfm.toString()); - Put put = new Put(regionInfo.getRegionName()); - put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, - Writables.getBytes(regionInfo)); - masterServices.getCatalogTracker().waitForMetaServerConnectionDefault().put( - CatalogTracker.META_REGION, put); - LOG.info("Updated region " + regionInfo + " to META"); + masterServices.getMasterFileSystem() + .createTableDescriptor(hrfm.getTableDesc()); + updateHRI(masterServices.getCatalogTracker() + .waitForMetaServerConnectionDefault(), + hrfm, CatalogTracker.META_REGION); return true; } }; @@ -289,6 +286,60 @@ public class MetaEditor { return htds; } + /** + * Migrate root and meta to newer version. This updates the META and ROOT + * and removes the HTD from HRI. + * @param masterServices + * @throws IOException + */ + public static void migrateRootAndMeta(final MasterServices masterServices) + throws IOException { + updateRootWithNewRegionInfo(masterServices); + updateMetaWithNewRegionInfo(masterServices); + } + + /** + * Update the ROOT with new HRI. (HRI with no HTD) + * @param masterServices + * @return + * @throws IOException + */ + public static List updateRootWithNewRegionInfo( + final MasterServices masterServices) + throws IOException { + final List htds = new ArrayList(); + Visitor v = new Visitor() { + @Override + public boolean visit(Result r) throws IOException { + if (r == null || r.isEmpty()) return true; + HRegionInfo090x hrfm = getHRegionInfoForMigration(r); + if (hrfm == null) return true; + htds.add(hrfm.getTableDesc()); + masterServices.getMasterFileSystem().createTableDescriptor( + hrfm.getTableDesc()); + updateHRI(masterServices.getCatalogTracker() + .waitForRootServerConnectionDefault(), + hrfm, CatalogTracker.ROOT_REGION); + return true; + } + }; + MetaReader.fullScan( + masterServices.getCatalogTracker().waitForRootServerConnectionDefault(), + v, HRegionInfo.ROOT_REGIONINFO.getRegionName(), null); + return htds; + } + + private static void updateHRI(HRegionInterface hRegionInterface, + HRegionInfo090x hRegionInfo090x, byte[] regionName) + throws IOException { + HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x); + Put put = new Put(regionInfo.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(regionInfo)); + hRegionInterface.put(regionName, put); + LOG.info("Updated region " + regionInfo + " to " + Bytes.toString(regionName)); + } + public static HRegionInfo090x getHRegionInfoForMigration( Result data) throws IOException { HRegionInfo090x info = null; Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1167006&r1=1167005&r2=1167006&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Fri Sep 9 05:25:24 2011 @@ -252,19 +252,8 @@ public class MetaReader { throws IOException { HRegionInterface metaServer = catalogTracker.waitForMetaServerConnectionDefault(); - Scan scan = new Scan(); - if (startrow != null) scan.setStartRow(startrow); - scan.addFamily(HConstants.CATALOG_FAMILY); - long scannerid = metaServer.openScanner( - HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan); - try { - Result data; - while((data = metaServer.next(scannerid)) != null) { - if (!data.isEmpty()) visitor.visit(data); - } - } finally { - metaServer.close(scannerid); - } + fullScan(metaServer, visitor, + HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), startrow); return; } @@ -622,7 +611,32 @@ public class MetaReader { return regions; } - + /** + * Fully scan a given region, on a given server starting with given row. + * @param hRegionInterface region server + * @param visitor visitor + * @param regionName name of region + * @param startrow start row + * @throws IOException + */ + public static void fullScan(HRegionInterface hRegionInterface, + Visitor visitor, final byte[] regionName, + byte[] startrow) throws IOException { + if (hRegionInterface == null) return; + Scan scan = new Scan(); + if (startrow != null) scan.setStartRow(startrow); + scan.addFamily(HConstants.CATALOG_FAMILY); + long scannerid = hRegionInterface.openScanner(regionName, scan); + try { + Result data; + while((data = hRegionInterface.next(scannerid)) != null) { + if (!data.isEmpty()) visitor.visit(data); + } + } finally { + hRegionInterface.close(scannerid); + } + return; + } /** Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1167006&r1=1167005&r2=1167006&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Fri Sep 9 05:25:24 2011 @@ -914,7 +914,8 @@ public class HConnectionManager { // possible we got a region of a different table... if (!Bytes.equals(regionInfo.getTableName(), tableName)) { throw new TableNotFoundException( - "Table '" + Bytes.toString(tableName) + "' was not found."); + "Table '" + Bytes.toString(tableName) + "' was not found, got: " + + Bytes.toString(regionInfo.getTableName()) + "."); } if (regionInfo.isSplit()) { throw new RegionOfflineException("the only available region for" + Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1167006&r1=1167005&r2=1167006&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Fri Sep 9 05:25:24 2011 @@ -526,14 +526,15 @@ implements HMasterInterface, HMasterRegi if (!isMetaHRIUpdated()) { LOG.info("Meta has HRI with HTDs. Updating meta now."); try { - MetaEditor.updateMetaWithNewRegionInfo(this); - LOG.info("Meta updated with new HRI."); + MetaEditor.migrateRootAndMeta(this); + LOG.info("ROOT and Meta updated with new HRI."); return true; } catch (IOException e) { - throw new RuntimeException("Update Meta with nw HRI failed. Master startup aborted."); + throw new RuntimeException("Update ROOT/Meta with new HRI failed." + + "Master startup aborted."); } } - LOG.info("Meta already up-to date with new HRI."); + LOG.info("ROOT/Meta already up-to date with new HRI."); return true; } @@ -977,7 +978,6 @@ implements HMasterInterface, HMasterRegi return hRegionInfos; } - private static boolean isCatalogTable(final byte [] tableName) { return Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) || Bytes.equals(tableName, HConstants.META_TABLE_NAME); Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1167006&r1=1167005&r2=1167006&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Fri Sep 9 05:25:24 2011 @@ -327,9 +327,18 @@ public class MasterFileSystem { if (!FSUtils.rootRegionExists(fs, rd)) { bootstrap(rd, c); } + createRootTableInfo(rd); return rd; } + private void createRootTableInfo(Path rd) throws IOException { + // Create ROOT tableInfo if required. + if (!FSUtils.tableInfoExists(fs, rd, + Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) { + FSUtils.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC, this.conf); + } + } + private static void bootstrap(final Path rd, final Configuration c) throws IOException { LOG.info("BOOTSTRAP: creating ROOT and first META regions"); Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java?rev=1167006&r1=1167005&r2=1167006&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java Fri Sep 9 05:25:24 2011 @@ -31,6 +31,9 @@ import org.apache.hadoop.hbase.HConstant import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableExistsException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + /** * Implementation of {@link TableDescriptors} that reads descriptors from the @@ -39,6 +42,8 @@ import org.apache.hadoop.hbase.TableExis * the filesystem or can be read and write. */ public class FSTableDescriptors implements TableDescriptors { + + private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class); private final FileSystem fs; private final Path rootdir; private final boolean fsreadonly; @@ -151,7 +156,14 @@ public class FSTableDescriptors implemen Map htds = new TreeMap(); List tableDirs = FSUtils.getTableDirs(fs, rootdir); for (Path d: tableDirs) { - HTableDescriptor htd = get(d.getName()); + HTableDescriptor htd = null; + try { + + htd = get(d.getName()); + } catch (FileNotFoundException fnfe) { + // inability of retrieving one HTD shouldn't stop getting the remaining + LOG.warn("Trouble retrieving htd", fnfe); + } if (htd == null) continue; htds.put(d.getName(), htd); } Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1167006&r1=1167005&r2=1167006&view=diff ============================================================================== --- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original) +++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Fri Sep 9 05:25:24 2011 @@ -490,6 +490,20 @@ public abstract class FSUtils { return fs.exists(rootRegionDir); } + /** + * Checks if .tableinfo exists for given table + * + * @param fs file system + * @param rootdir root directory of HBase installation + * @param tableName name of table + * @return true if exists + * @throws IOException + */ + public static boolean tableInfoExists(FileSystem fs, Path rootdir, + String tableName) throws IOException { + Path tablePath = getTableInfoPath(rootdir, tableName); + return fs.exists(tablePath); + } /** * Compute HDFS blocks distribution of a given file, or a portion of the file