Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hbase.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1523867 - in /hbase/trunk/hbase-server/src:
 main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java
 main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
 test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java
Date: Tue, 17 Sep 2013 02:10:34 -0000
To: commits@hbase.apache.org
From: stack@apache.org
Message-Id: <20130917021035.21C6C23889D5@eris.apache.org>

Author: stack
Date: Tue Sep 17 02:10:34 2013
New Revision: 1523867

URL: http://svn.apache.org/r1523867
Log:
HBASE-9539 Handle post namespace snapshot files when checking for HFile V1

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java?rev=1523867&r1=1523866&r2=1523867&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/migration/UpgradeTo96.java Tue Sep 17 02:10:34 2013
@@ -120,12 +120,12 @@ public class UpgradeTo96 extends Configu
       if (res == 0) LOG.info("No HFileV1 found.");
       else {
         LOG.warn("There are some HFileV1, or corrupt files (files with incorrect major version).");
-        return -1;
       }
+      return res;
     }
     // if the user wants to upgrade, check for any HBase live process.
     // If yes, prompt the user to stop them
-    if (upgrade) {
+    else if (upgrade) {
       if (isAnyHBaseProcessAlive()) {
         LOG.error("Some HBase processes are still alive, or znodes not expired yet. "
             + "Please stop them before upgrade or try after some time.");

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java?rev=1523867&r1=1523866&r2=1523867&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileV1Detector.java Tue Sep 17 02:10:34 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -45,6 +46,9 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.io.FileLink;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
@@ -67,6 +71,14 @@ public class HFileV1Detector extends Con
   private FileSystem fs;
   private static final Log LOG = LogFactory.getLog(HFileV1Detector.class);
   private static final int DEFAULT_NUM_OF_THREADS = 10;
+  /**
+   * Pre-namespace archive directory
+   */
+  private static final String PRE_NS_DOT_ARCHIVE = ".archive";
+  /**
+   * Pre-namespace tmp directory
+   */
+  private static final String PRE_NS_DOT_TMP = ".tmp";
   private int numOfThreads;
   /**
    * directory to start the processing.
@@ -93,6 +105,10 @@ public class HFileV1Detector extends Con
       .newSetFromMap(new ConcurrentHashMap<Path, Boolean>());
 
   private Options options = new Options();
+  /**
+   * used for computing pre-namespace paths for hfilelinks
+   */
+  private Path defaultNamespace;
 
   public HFileV1Detector() {
     Option pathOption = new Option("p", "path", true, "Path to a table, or hbase installation");
@@ -171,6 +187,11 @@ public class HFileV1Detector extends Con
     return -1;
   }
 
+  private void setDefaultNamespaceDir() throws IOException {
+    Path dataDir = new Path(FSUtils.getRootDir(getConf()), HConstants.BASE_NAMESPACE_DIR);
+    defaultNamespace = new Path(dataDir, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
+  }
+
   private int processResult(Set<Path> regionsWithHFileV1) {
     LOG.info("Result: \n");
     printSet(processedTables, "Tables Processed: ");
@@ -292,9 +313,9 @@ public class HFileV1Detector extends Con
               if (StoreFileInfo.isReference(storeFilePath)) continue;
               // check whether this path is a HFileLink.
               else if (HFileLink.isHFileLink(storeFilePath)) {
-                HFileLink fileLink = new HFileLink(getConf(), storeFilePath);
-                fsdis = fileLink.open(fs);
-                lenToRead = fileLink.getFileStatus(fs).getLen();
+                FileLink fLink = getFileLinkWithPreNSPath(storeFilePath);
+                fsdis = fLink.open(fs);
+                lenToRead = fLink.getFileStatus(fs).getLen();
               } else {
                 // a regular hfile
                 fsdis = fs.open(storeFilePath);
@@ -337,6 +358,47 @@ public class HFileV1Detector extends Con
     return f;
   }
 
+  /**
+   * Creates a FileLink which adds pre-namespace paths in its list of available paths. This is used
+   * when reading a snapshot file in a pre-namespace file layout, for example, while upgrading.
+   * @param storeFilePath
+   * @return a FileLink which could read from pre-namespace paths.
+   * @throws IOException
+   */
+  public FileLink getFileLinkWithPreNSPath(Path storeFilePath) throws IOException {
+    HFileLink link = new HFileLink(getConf(), storeFilePath);
+    List<Path> pathsToProcess = getPreNSPathsForHFileLink(link);
+    pathsToProcess.addAll(Arrays.asList(link.getLocations()));
+    return new FileLink(pathsToProcess);
+  }
+
+  private List<Path> getPreNSPathsForHFileLink(HFileLink fileLink) throws IOException {
+    if (defaultNamespace == null) setDefaultNamespaceDir();
+    List<Path> p = new ArrayList<Path>();
+    String relativeTablePath = removeDefaultNSPath(fileLink.getOriginPath());
+    p.add(getPreNSPath(PRE_NS_DOT_ARCHIVE, relativeTablePath));
+    p.add(getPreNSPath(PRE_NS_DOT_TMP, relativeTablePath));
+    p.add(getPreNSPath(null, relativeTablePath));
+    return p;
+  }
+
+  /**
+   * Removes the prefix of defaultNamespace from the path.
+   * @param originPath
+   * @return
+   */
+  private String removeDefaultNSPath(Path originalPath) {
+    String pathStr = originalPath.toString();
+    if (!pathStr.startsWith(defaultNamespace.toString())) return pathStr;
+    return pathStr.substring(defaultNamespace.toString().length() + 1);
+  }
+
+  private Path getPreNSPath(String prefix, String relativeTablePath) throws IOException {
+    String relativePath = (prefix == null ? relativeTablePath : prefix + Path.SEPARATOR
+        + relativeTablePath);
+    return new Path(FSUtils.getRootDir(getConf()), relativePath);
+  }
+
   private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
     // check for old format, of having /table/.tableinfo; hbase:meta doesn't has .tableinfo,
     // include it.

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java?rev=1523867&r1=1523866&r2=1523867&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java Tue Sep 17 02:10:34 2013
@@ -17,24 +17,22 @@
  */
 package org.apache.hadoop.hbase.migration;
 
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.IOException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.master.TableNamespaceManager;
+import org.apache.hadoop.hbase.io.FileLink;
+import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer;
@@ -43,20 +41,14 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.HFileV1Detector;
-import org.apache.hadoop.hbase.util.ZKDataMigrator;
-import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.server.ZKDatabase;
-import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
-import org.mockito.Mockito;
 
 import com.google.protobuf.InvalidProtocolBufferException;
 
@@ -198,6 +190,35 @@ public class TestUpgradeTo96 {
   }
 
   @Test
+  public void testHFileLink() throws Exception {
+    // pass a link, and verify that correct paths are returned.
+    Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration());
+    Path aFileLink = new Path(rootDir, "table/2086db948c48/cf/table=21212abcdc33-0906db948c48");
+    Path preNamespaceTablePath = new Path(rootDir, "table/21212abcdc33/cf/0906db948c48");
+    Path preNamespaceArchivePath =
+      new Path(rootDir, ".archive/table/21212abcdc33/cf/0906db948c48");
+    Path preNamespaceTempPath = new Path(rootDir, ".tmp/table/21212abcdc33/cf/0906db948c48");
+    boolean preNSTablePathExists = false;
+    boolean preNSArchivePathExists = false;
+    boolean preNSTempPathExists = false;
+    assertTrue(HFileLink.isHFileLink(aFileLink));
+    HFileLink hFileLink = new HFileLink(TEST_UTIL.getConfiguration(), aFileLink);
+    assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString()));
+
+    HFileV1Detector t = new HFileV1Detector();
+    t.setConf(TEST_UTIL.getConfiguration());
+    FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink);
+    //assert it has 6 paths (2 NS, 2 Pre NS, and 2 .tmp)  to look.
+    assertTrue(fileLink.getLocations().length == 6);
+    for (Path p : fileLink.getLocations()) {
+      if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true;
+      if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true;
+      if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true;
+    }
+    assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists);
+  }
+
+  @Test
   public void testADirForHFileV1() throws Exception {
     Path tablePath = new Path(hbaseRootDir, "foo");
     System.out.println("testADirForHFileV1: " + tablePath.makeQualified(fs));