hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bus...@apache.org
Subject [3/8] hbase git commit: HBASE-16904 Snapshot related changes for FS redo work
Date Wed, 30 Nov 2016 06:56:21 GMT
http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
deleted file mode 100644
index 8c24b1e..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
+++ /dev/null
@@ -1,823 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.snapshot;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ThreadPoolExecutor;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.MetaTableAccessor;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
-import org.apache.hadoop.hbase.fs.RegionStorage;
-import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
-import org.apache.hadoop.hbase.fs.legacy.io.HFileLink;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.ModifyRegionUtils;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.io.IOUtils;
-
-/**
- * Helper to Restore/Clone a Snapshot
- *
- * <p>The helper assumes that a table is already created, and by calling restore()
- * the content present in the snapshot will be restored as the new content of the table.
- *
- * <p>Clone from Snapshot: If the target table is empty, the restore operation
- * is just a "clone operation", where the only operations are:
- * <ul>
- *  <li>for each region in the snapshot create a new region
- *    (note that the region will have a different name, since the encoding contains the table name)
- *  <li>for each file in the region create a new HFileLink to point to the original file.
- *  <li>restore the logs, if any
- * </ul>
- *
- * <p>Restore from Snapshot:
- * <ul>
- *  <li>for each region in the table verify which are available in the snapshot and which are not
- *    <ul>
- *    <li>if the region is not present in the snapshot, remove it.
- *    <li>if the region is present in the snapshot
- *      <ul>
- *      <li>for each file in the table region verify which are available in the snapshot
- *        <ul>
- *          <li>if the hfile is not present in the snapshot, remove it
- *          <li>if the hfile is present, keep it (nothing to do)
- *        </ul>
- *      <li>for each file in the snapshot region but not in the table
- *        <ul>
- *          <li>create a new HFileLink that point to the original file
- *        </ul>
- *      </ul>
- *    </ul>
- *  <li>for each region in the snapshot not present in the current table state
- *    <ul>
- *    <li>create a new region and for each file in the region create a new HFileLink
- *      (This is the same as the clone operation)
- *    </ul>
- *  <li>restore the logs, if any
- * </ul>
- *
- * TODO update for MasterStorage / RegionStorage
- */
-@InterfaceAudience.Private
-public class RestoreSnapshotHelper {
-  private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
-
-  private final Map<byte[], byte[]> regionsMap =
-        new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
-
-  private final Map<String, Pair<String, String> > parentsMap =
-      new HashMap<String, Pair<String, String> >();
-
-  private final ForeignExceptionDispatcher monitor;
-  private final MonitoredTask status;
-
-  private final SnapshotManifest snapshotManifest;
-  private final SnapshotDescription snapshotDesc;
-  private final TableName snapshotTable;
-
-  private final HTableDescriptor tableDesc;
-  private final Path tableDir;
-
-  private final Configuration conf;
-  private final FileSystem fs;
-  private final boolean createBackRefs;
-
-  public RestoreSnapshotHelper(final Configuration conf, final SnapshotManifest manifest,
-      final HTableDescriptor tableDescriptor, final ForeignExceptionDispatcher monitor,
-      final MonitoredTask status) throws IOException {
-    this(conf, manifest, tableDescriptor, monitor, status, true);
-  }
-
-  public RestoreSnapshotHelper(final Configuration conf, final SnapshotManifest manifest,
-      final HTableDescriptor tableDescriptor, final ForeignExceptionDispatcher monitor,
-      final MonitoredTask status, final boolean createBackRefs) throws IOException {
-    this.fs = FSUtils.getCurrentFileSystem(conf);
-    this.conf = conf;
-    this.snapshotManifest = manifest;
-    this.snapshotDesc = manifest.getSnapshotDescription();
-    this.snapshotTable = TableName.valueOf(snapshotDesc.getTable());
-    this.tableDesc = tableDescriptor;
-    this.tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), tableDesc.getTableName());
-    this.monitor = monitor;
-    this.status = status;
-    this.createBackRefs = createBackRefs;
-  }
-
-  /**
-   * Restore the on-disk table to a specified snapshot state.
-   * @return the set of regions touched by the restore operation
-   */
-  public RestoreMetaChanges restoreStorageRegions() throws IOException {
-    ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot");
-    try {
-      return restoreHdfsRegions(exec);
-    } finally {
-      exec.shutdown();
-    }
-  }
-
-  private RestoreMetaChanges restoreHdfsRegions(final ThreadPoolExecutor exec) throws IOException {
-    LOG.info("starting restore table regions using snapshot=" + snapshotDesc);
-
-    Map<String, SnapshotRegionManifest> regionManifests = snapshotManifest.getRegionManifestsMap();
-    if (regionManifests == null) {
-      LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
-      return null;
-    }
-
-    RestoreMetaChanges metaChanges = new RestoreMetaChanges(tableDesc, parentsMap);
-
-    // Take a copy of the manifest.keySet() since we are going to modify
-    // this instance, by removing the regions already present in the restore dir.
-    Set<String> regionNames = new HashSet<String>(regionManifests.keySet());
-
-    HRegionInfo mobRegion = MobUtils.getMobRegionInfo(snapshotManifest.getTableDescriptor()
-        .getTableName());
-    // Identify which region are still available and which not.
-    // NOTE: we rely upon the region name as: "table name, start key, end key"
-    List<HRegionInfo> tableRegions = getTableRegions();
-    if (tableRegions != null) {
-      monitor.rethrowException();
-      for (HRegionInfo regionInfo: tableRegions) {
-        String regionName = regionInfo.getEncodedName();
-        if (regionNames.contains(regionName)) {
-          LOG.info("region to restore: " + regionName);
-          regionNames.remove(regionName);
-          metaChanges.addRegionToRestore(regionInfo);
-        } else {
-          LOG.info("region to remove: " + regionName);
-          metaChanges.addRegionToRemove(regionInfo);
-        }
-      }
-
-      // Restore regions using the snapshot data
-      monitor.rethrowException();
-      status.setStatus("Restoring table regions...");
-      if (regionNames.contains(mobRegion.getEncodedName())) {
-        // restore the mob region in case
-        List<HRegionInfo> mobRegions = new ArrayList<HRegionInfo>(1);
-        mobRegions.add(mobRegion);
-        restoreHdfsMobRegions(exec, regionManifests, mobRegions);
-        regionNames.remove(mobRegion.getEncodedName());
-      }
-      restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore());
-      status.setStatus("Finished restoring all table regions.");
-
-      // Remove regions from the current table
-      monitor.rethrowException();
-      status.setStatus("Starting to delete excess regions from table");
-      removeHdfsRegions(exec, metaChanges.getRegionsToRemove());
-      status.setStatus("Finished deleting excess regions from table.");
-    }
-
-    // Regions to Add: present in the snapshot but not in the current table
-    if (regionNames.size() > 0) {
-      List<HRegionInfo> regionsToAdd = new ArrayList<HRegionInfo>(regionNames.size());
-
-      monitor.rethrowException();
-      // add the mob region
-      if (regionNames.contains(mobRegion.getEncodedName())) {
-        cloneHdfsMobRegion(regionManifests, mobRegion);
-        regionNames.remove(mobRegion.getEncodedName());
-      }
-      for (String regionName: regionNames) {
-        LOG.info("region to add: " + regionName);
-        regionsToAdd.add(HRegionInfo.convert(regionManifests.get(regionName).getRegionInfo()));
-      }
-
-      // Create new regions cloning from the snapshot
-      monitor.rethrowException();
-      status.setStatus("Cloning regions...");
-      HRegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd);
-      metaChanges.setNewRegions(clonedRegions);
-      status.setStatus("Finished cloning regions.");
-    }
-
-    LOG.info("finishing restore table regions using snapshot=" + snapshotDesc);
-
-    return metaChanges;
-  }
-
-  /**
-   * Describe the set of operations needed to update hbase:meta after restore.
-   */
-  public static class RestoreMetaChanges {
-    private final Map<String, Pair<String, String> > parentsMap;
-    private final HTableDescriptor htd;
-
-    private List<HRegionInfo> regionsToRestore = null;
-    private List<HRegionInfo> regionsToRemove = null;
-    private List<HRegionInfo> regionsToAdd = null;
-
-    public RestoreMetaChanges(HTableDescriptor htd, Map<String, Pair<String, String> > parentsMap) {
-      this.parentsMap = parentsMap;
-      this.htd = htd;
-    }
-
-    public HTableDescriptor getTableDescriptor() {
-      return htd;
-    }
-
-    /**
-     * Returns the map of parent-children_pair.
-     * @return the map
-     */
-    public Map<String, Pair<String, String>> getParentToChildrenPairMap() {
-      return this.parentsMap;
-    }
-
-    /**
-     * @return true if there're new regions
-     */
-    public boolean hasRegionsToAdd() {
-      return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
-    }
-
-    /**
-     * Returns the list of new regions added during the on-disk restore.
-     * The caller is responsible to add the regions to META.
-     * e.g MetaTableAccessor.addRegionsToMeta(...)
-     * @return the list of regions to add to META
-     */
-    public List<HRegionInfo> getRegionsToAdd() {
-      return this.regionsToAdd;
-    }
-
-    /**
-     * @return true if there're regions to restore
-     */
-    public boolean hasRegionsToRestore() {
-      return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
-    }
-
-    /**
-     * Returns the list of 'restored regions' during the on-disk restore.
-     * The caller is responsible to add the regions to hbase:meta if not present.
-     * @return the list of regions restored
-     */
-    public List<HRegionInfo> getRegionsToRestore() {
-      return this.regionsToRestore;
-    }
-
-    /**
-     * @return true if there're regions to remove
-     */
-    public boolean hasRegionsToRemove() {
-      return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
-    }
-
-    /**
-     * Returns the list of regions removed during the on-disk restore.
-     * The caller is responsible to remove the regions from META.
-     * e.g. MetaTableAccessor.deleteRegions(...)
-     * @return the list of regions to remove from META
-     */
-    public List<HRegionInfo> getRegionsToRemove() {
-      return this.regionsToRemove;
-    }
-
-    void setNewRegions(final HRegionInfo[] hris) {
-      if (hris != null) {
-        regionsToAdd = Arrays.asList(hris);
-      } else {
-        regionsToAdd = null;
-      }
-    }
-
-    void addRegionToRemove(final HRegionInfo hri) {
-      if (regionsToRemove == null) {
-        regionsToRemove = new LinkedList<HRegionInfo>();
-      }
-      regionsToRemove.add(hri);
-    }
-
-    void addRegionToRestore(final HRegionInfo hri) {
-      if (regionsToRestore == null) {
-        regionsToRestore = new LinkedList<HRegionInfo>();
-      }
-      regionsToRestore.add(hri);
-    }
-
-    public void updateMetaParentRegions(Connection connection,
-        final List<HRegionInfo> regionInfos) throws IOException {
-      if (regionInfos == null || parentsMap.isEmpty()) return;
-
-      // Extract region names and offlined regions
-      Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
-      List<HRegionInfo> parentRegions = new LinkedList<>();
-      for (HRegionInfo regionInfo: regionInfos) {
-        if (regionInfo.isSplitParent()) {
-          parentRegions.add(regionInfo);
-        } else {
-          regionsByName.put(regionInfo.getEncodedName(), regionInfo);
-        }
-      }
-
-      // Update Offline parents
-      for (HRegionInfo regionInfo: parentRegions) {
-        Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
-        if (daughters == null) {
-          // The snapshot contains an unreferenced region.
-          // It will be removed by the CatalogJanitor.
-          LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
-          continue;
-        }
-
-        // One side of the split is already compacted
-        if (daughters.getSecond() == null) {
-          daughters.setSecond(daughters.getFirst());
-        }
-
-        LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
-        MetaTableAccessor.addRegionToMeta(connection, regionInfo,
-          regionsByName.get(daughters.getFirst()),
-          regionsByName.get(daughters.getSecond()));
-      }
-    }
-  }
-
-  /**
-   * Remove specified regions from the file-system, using the archiver.
-   */
-  private void removeHdfsRegions(final ThreadPoolExecutor exec, final List<HRegionInfo> regions)
-      throws IOException {
-    if (regions == null || regions.size() == 0) return;
-    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
-      @Override
-      public void editRegion(final HRegionInfo hri) throws IOException {
-        HFileArchiver.archiveRegion(conf, fs, hri);
-      }
-    });
-  }
-
-  /**
-   * Restore specified regions by restoring content to the snapshot state.
-   */
-  private void restoreHdfsRegions(final ThreadPoolExecutor exec,
-      final Map<String, SnapshotRegionManifest> regionManifests,
-      final List<HRegionInfo> regions) throws IOException {
-    if (regions == null || regions.size() == 0) return;
-    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
-      @Override
-      public void editRegion(final HRegionInfo hri) throws IOException {
-        restoreRegion(hri, regionManifests.get(hri.getEncodedName()));
-      }
-    });
-  }
-
-  /**
-   * Restore specified mob regions by restoring content to the snapshot state.
-   */
-  private void restoreHdfsMobRegions(final ThreadPoolExecutor exec,
-      final Map<String, SnapshotRegionManifest> regionManifests,
-      final List<HRegionInfo> regions) throws IOException {
-    if (regions == null || regions.size() == 0) return;
-    ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
-      @Override
-      public void editRegion(final HRegionInfo hri) throws IOException {
-        restoreMobRegion(hri, regionManifests.get(hri.getEncodedName()));
-      }
-    });
-  }
-
-  private Map<String, List<SnapshotRegionManifest.StoreFile>> getRegionHFileReferences(
-      final SnapshotRegionManifest manifest) {
-    Map<String, List<SnapshotRegionManifest.StoreFile>> familyMap =
-      new HashMap<String, List<SnapshotRegionManifest.StoreFile>>(manifest.getFamilyFilesCount());
-    for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
-      familyMap.put(familyFiles.getFamilyName().toStringUtf8(),
-        new ArrayList<SnapshotRegionManifest.StoreFile>(familyFiles.getStoreFilesList()));
-    }
-    return familyMap;
-  }
-
-  /**
-   * Restore region by removing files not in the snapshot
-   * and adding the missing ones from the snapshot.
-   */
-  private void restoreRegion(final HRegionInfo regionInfo,
-      final SnapshotRegionManifest regionManifest) throws IOException {
-    restoreRegion(regionInfo, regionManifest, new Path(tableDir, regionInfo.getEncodedName()));
-  }
-
-  /**
-   * Restore mob region by removing files not in the snapshot
-   * and adding the missing ones from the snapshot.
-   */
-  private void restoreMobRegion(final HRegionInfo regionInfo,
-      final SnapshotRegionManifest regionManifest) throws IOException {
-    if (regionManifest == null) {
-      return;
-    }
-    restoreRegion(regionInfo, regionManifest,
-      MobUtils.getMobRegionPath(conf, tableDesc.getTableName()));
-  }
-
-  /**
-   * Restore region by removing files not in the snapshot
-   * and adding the missing ones from the snapshot.
-   */
-  private void restoreRegion(final HRegionInfo regionInfo,
-      final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException {
-    Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles =
-                getRegionHFileReferences(regionManifest);
-
-    String tableName = tableDesc.getTableName().getNameAsString();
-
-    // Restore families present in the table
-    for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
-      byte[] family = Bytes.toBytes(familyDir.getName());
-      Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
-      List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles =
-          snapshotFiles.remove(familyDir.getName());
-      if (snapshotFamilyFiles != null) {
-        List<SnapshotRegionManifest.StoreFile> hfilesToAdd =
-            new ArrayList<SnapshotRegionManifest.StoreFile>();
-        for (SnapshotRegionManifest.StoreFile storeFile: snapshotFamilyFiles) {
-          if (familyFiles.contains(storeFile.getName())) {
-            // HFile already present
-            familyFiles.remove(storeFile.getName());
-          } else {
-            // HFile missing
-            hfilesToAdd.add(storeFile);
-          }
-        }
-
-        // Remove hfiles not present in the snapshot
-        for (String hfileName: familyFiles) {
-          Path hfile = new Path(familyDir, hfileName);
-          LOG.trace("Removing hfile=" + hfileName +
-            " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
-          HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
-        }
-
-        // Restore Missing files
-        for (SnapshotRegionManifest.StoreFile storeFile: hfilesToAdd) {
-          LOG.debug("Adding HFileLink " + storeFile.getName() +
-            " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
-          restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
-        }
-      } else {
-        // Family doesn't exists in the snapshot
-        LOG.trace("Removing family=" + Bytes.toString(family) +
-          " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
-        HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
-        fs.delete(familyDir, true);
-      }
-    }
-
-    // Add families not present in the table
-    for (Map.Entry<String, List<SnapshotRegionManifest.StoreFile>> familyEntry:
-                                                                      snapshotFiles.entrySet()) {
-      Path familyDir = new Path(regionDir, familyEntry.getKey());
-      if (!fs.mkdirs(familyDir)) {
-        throw new IOException("Unable to create familyDir=" + familyDir);
-      }
-
-      for (SnapshotRegionManifest.StoreFile storeFile: familyEntry.getValue()) {
-        LOG.trace("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
-        restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
-      }
-    }
-  }
-
-  /**
-   * @return The set of files in the specified family directory.
-   */
-  private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
-    FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
-    if (hfiles == null) return Collections.emptySet();
-
-    Set<String> familyFiles = new HashSet<String>(hfiles.length);
-    for (int i = 0; i < hfiles.length; ++i) {
-      String hfileName = hfiles[i].getPath().getName();
-      familyFiles.add(hfileName);
-    }
-
-    return familyFiles;
-  }
-
-  /**
-   * Clone specified regions. For each region create a new region
-   * and create a HFileLink for each hfile.
-   */
-  private HRegionInfo[] cloneHdfsRegions(final ThreadPoolExecutor exec,
-      final Map<String, SnapshotRegionManifest> regionManifests,
-      final List<HRegionInfo> regions) throws IOException {
-    if (regions == null || regions.size() == 0) return null;
-
-    final Map<String, HRegionInfo> snapshotRegions =
-      new HashMap<String, HRegionInfo>(regions.size());
-
-    // clone region info (change embedded tableName with the new one)
-    HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
-    for (int i = 0; i < clonedRegionsInfo.length; ++i) {
-      // clone the region info from the snapshot region info
-      HRegionInfo snapshotRegionInfo = regions.get(i);
-      clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
-
-      // add the region name mapping between snapshot and cloned
-      String snapshotRegionName = snapshotRegionInfo.getEncodedName();
-      String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
-      regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
-      LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
-
-      // Add mapping between cloned region name and snapshot region info
-      snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
-    }
-
-    // create the regions on disk
-    ModifyRegionUtils.createRegions(exec, conf,
-        tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
-        @Override
-        public void fillRegion(final HRegion region) throws IOException {
-          HRegionInfo snapshotHri = snapshotRegions.get(region.getRegionInfo().getEncodedName());
-          cloneRegion(region, snapshotHri, regionManifests.get(snapshotHri.getEncodedName()));
-        }
-      });
-
-    return clonedRegionsInfo;
-  }
-
-  /**
-   * Clone the mob region. For the region create a new region
-   * and create a HFileLink for each hfile.
-   */
-  private void cloneHdfsMobRegion(final Map<String, SnapshotRegionManifest> regionManifests,
-      final HRegionInfo region) throws IOException {
-    // clone region info (change embedded tableName with the new one)
-    Path clonedRegionPath = MobUtils.getMobRegionPath(conf, tableDesc.getTableName());
-    cloneRegion(clonedRegionPath, region, regionManifests.get(region.getEncodedName()));
-  }
-
-  /**
-   * Clone region directory content from the snapshot info.
-   *
-   * Each region is encoded with the table name, so the cloned region will have
-   * a different region name.
-   *
-   * Instead of copying the hfiles a HFileLink is created.
-   *
-   * @param regionDir {@link Path} cloned dir
-   * @param snapshotRegionInfo
-   */
-  private void cloneRegion(final Path regionDir, final HRegionInfo snapshotRegionInfo,
-      final SnapshotRegionManifest manifest) throws IOException {
-    final String tableName = tableDesc.getTableName().getNameAsString();
-    for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) {
-      Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8());
-      for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) {
-        LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName);
-        restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs);
-      }
-    }
-  }
-
-  /**
-   * Clone region directory content from the snapshot info.
-   *
-   * Each region is encoded with the table name, so the cloned region will have
-   * a different region name.
-   *
-   * Instead of copying the hfiles a HFileLink is created.
-   *
-   * @param region {@link HRegion} cloned
-   * @param snapshotRegionInfo
-   */
-  private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo,
-      final SnapshotRegionManifest manifest) throws IOException {
-    cloneRegion(new Path(tableDir, region.getRegionInfo().getEncodedName()), snapshotRegionInfo,
-      manifest);
-  }
-
-  /**
-   * Create a new {@link HFileLink} to reference the store file.
-   * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference.
-   * <ul>
-   *   <li>hfile: abc -> table=region-abc
-   *   <li>reference: abc.1234 -> table=region-abc.1234
-   *   <li>hfilelink: table=region-hfile -> table=region-hfile
-   * </ul>
-   * @param familyDir destination directory for the store file
-   * @param regionInfo destination region info for the table
-   * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @param storeFile store file name (can be a Reference, HFileLink or simple HFile)
-   */
-  private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
-      final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef)
-          throws IOException {
-    String hfileName = storeFile.getName();
-    if (HFileLink.isHFileLink(hfileName)) {
-      HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
-    } else if (StoreFileInfo.isReference(hfileName)) {
-      restoreReferenceFile(familyDir, regionInfo, storeFile);
-    } else {
-      HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
-    }
-  }
-
-  /**
-   * Create a new {@link Reference} as copy of the source one.
-   * <p><blockquote><pre>
-   * The source table looks like:
-   *    1234/abc      (original file)
-   *    5678/abc.1234 (reference file)
-   *
-   * After the clone operation looks like:
-   *   wxyz/table=1234-abc
-   *   stuv/table=1234-abc.wxyz
-   *
-   * NOTE that the region name in the clone changes (md5 of regioninfo)
-   * and the reference should reflect that change.
-   * </pre></blockquote>
-   * @param familyDir destination directory for the store file
-   * @param regionInfo destination region info for the table
-   * @param storeFile reference file name
-   */
-  private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
-      final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
-    String hfileName = storeFile.getName();
-
-    // Extract the referred information (hfile name and parent region)
-    Path refPath =
-        StoreFileInfo.getReferredToFile(new Path(new Path(new Path(new Path(snapshotTable
-            .getNamespaceAsString(), snapshotTable.getQualifierAsString()), regionInfo
-            .getEncodedName()), familyDir.getName()), hfileName));
-    String snapshotRegionName = refPath.getParent().getParent().getName();
-    String fileName = refPath.getName();
-
-    // The new reference should have the cloned region name as parent, if it is a clone.
-    String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
-    if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
-
-    // The output file should be a reference link table=snapshotRegion-fileName.clonedRegionName
-    Path linkPath = null;
-    String refLink = fileName;
-    if (!HFileLink.isHFileLink(fileName)) {
-      refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
-      linkPath = new Path(familyDir,
-        HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
-    }
-
-    Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
-
-    // Create the new reference
-    if (storeFile.hasReference()) {
-      Reference reference = Reference.convert(storeFile.getReference());
-      reference.write(fs, outPath);
-    } else {
-      InputStream in;
-      if (linkPath != null) {
-        in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
-      } else {
-        linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
-                        regionInfo.getEncodedName()), familyDir.getName()), hfileName);
-        in = fs.open(linkPath);
-      }
-      OutputStream out = fs.create(outPath);
-      IOUtils.copyBytes(in, out, conf);
-    }
-
-    // Add the daughter region to the map
-    String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
-    LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
-    synchronized (parentsMap) {
-      Pair<String, String> daughters = parentsMap.get(clonedRegionName);
-      if (daughters == null) {
-        daughters = new Pair<String, String>(regionName, null);
-        parentsMap.put(clonedRegionName, daughters);
-      } else if (!regionName.equals(daughters.getFirst())) {
-        daughters.setSecond(regionName);
-      }
-    }
-  }
-
-  /**
-   * Create a new {@link HRegionInfo} from the snapshot region info.
-   * Keep the same startKey, endKey, regionId and split information but change
-   * the table name.
-   *
-   * @param snapshotRegionInfo Info for region to clone.
-   * @return the new HRegion instance
-   */
-  public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
-    return cloneRegionInfo(tableDesc.getTableName(), snapshotRegionInfo);
-  }
-
-  public static HRegionInfo cloneRegionInfo(TableName tableName, HRegionInfo snapshotRegionInfo) {
-    HRegionInfo regionInfo = new HRegionInfo(tableName,
-                      snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
-                      snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
-    regionInfo.setOffline(snapshotRegionInfo.isOffline());
-    return regionInfo;
-  }
-
-  /**
-   * @return the set of the regions contained in the table
-   */
-  private List<HRegionInfo> getTableRegions() throws IOException {
-    LOG.debug("get table regions: " + tableDir);
-    FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
-    if (regionDirs == null) return null;
-
-    List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
-    for (FileStatus regionDir: regionDirs) {
-      final RegionStorage rs = RegionStorage.open(conf, new LegacyPathIdentifier(regionDir.getPath()), false);
-      regions.add(rs.getRegionInfo());
-    }
-    LOG.debug("found " + regions.size() + " regions for table=" +
-        tableDesc.getTableName().getNameAsString());
-    return regions;
-  }
-
-  /**
-   * Copy the snapshot files for a snapshot scanner, discards meta changes.
-   * @param conf
-   * @param fs
-   * @param rootDir
-   * @param restoreDir
-   * @param snapshotName
-   * @throws IOException
-   */
-  public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, FileSystem fs,
-      Path rootDir, Path restoreDir, String snapshotName) throws IOException {
-    // ensure that restore dir is not under root dir
-    if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
-      throw new IllegalArgumentException("Filesystems for restore directory and HBase root " +
-          "directory should be the same");
-    }
-    if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath())) {
-      throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " +
-          "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
-    }
-
-    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
-    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
-    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
-
-    MonitoredTask status = TaskMonitor.get().createStatus(
-        "Restoring  snapshot '" + snapshotName + "' to directory " + restoreDir);
-    ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
-
-    // we send createBackRefs=false so that restored hfiles do not create back reference links
-    // in the base hbase root dir.
-    RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf,
-        manifest, manifest.getTableDescriptor(), monitor, status, false);
-    RestoreMetaChanges metaChanges = helper.restoreStorageRegions(); // TODO: parallelize.
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Restored table dir:" + restoreDir);
-      FSUtils.logFileSystemState(fs, restoreDir, LOG);
-    }
-    return metaChanges;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index 2fd619d..ce12ba9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -17,79 +17,21 @@
  */
 package org.apache.hadoop.hbase.snapshot;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Collections;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifestV2;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
 
 /**
  * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}.
- * <p>
- * Snapshots are laid out on disk like this:
- *
- * <pre>
- * /hbase/.snapshots
- *          /.tmp                &lt;---- working directory
- *          /[snapshot name]     &lt;----- completed snapshot
- * </pre>
- *
- * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc.
- * signified by '...' on the same directory depth).
- *
- * <pre>
- * /hbase/.snapshots/completed
- *                   .snapshotinfo          &lt;--- Description of the snapshot
- *                   .tableinfo             &lt;--- Copy of the tableinfo
- *                    /.logs
- *                        /[server_name]
- *                            /... [log files]
- *                         ...
- *                   /[region name]           &lt;---- All the region's information
- *                   .regioninfo              &lt;---- Copy of the HRegionInfo
- *                      /[column family name]
- *                          /[hfile name]     &lt;--- name of the hfile in the real region
- *                          ...
- *                      ...
- *                    ...
- * </pre>
- *
- * Utility methods in this class are useful for getting the correct locations for different parts of
- * the snapshot, as well as moving completed snapshots into place (see
- * {@link #completeSnapshot}, and writing the
- * {@link SnapshotDescription} to the working snapshot directory.
  */
 @InterfaceAudience.Private
 public final class SnapshotDescriptionUtils {
 
-  /**
-   * Filter that only accepts completed snapshot directories
-   */
-  public static class CompletedSnaphotDirectoriesFilter extends FSUtils.BlackListDirFilter {
-
-    /**
-     * @param fs
-     */
-    public CompletedSnaphotDirectoriesFilter(FileSystem fs) {
-      super(fs, Collections.singletonList(SNAPSHOT_TMP_DIR_NAME));
-    }
-  }
-
   private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class);
   /**
    * Version of the fs layout for a snapshot. Future snapshots may have different file layouts,
@@ -97,17 +39,6 @@ public final class SnapshotDescriptionUtils {
    */
   public static final int SNAPSHOT_LAYOUT_VERSION = SnapshotManifestV2.DESCRIPTOR_VERSION;
 
-  // snapshot directory constants
-  /**
-   * The file contains the snapshot basic information and it is under the directory of a snapshot.
-   */
-  public static final String SNAPSHOTINFO_FILE = ".snapshotinfo";
-
-  /** Temporary directory under the snapshot directory to store in-progress snapshots */
-  public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp";
-
-  /** This tag will be created in in-progess snapshots */
-  public static final String SNAPSHOT_IN_PROGRESS = ".inprogress";
   // snapshot operation values
   /** Default value if no start time is specified */
   public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0;
@@ -157,86 +88,6 @@ public final class SnapshotDescriptionUtils {
   }
 
   /**
-   * Get the snapshot root directory. All the snapshots are kept under this directory, i.e.
-   * ${hbase.rootdir}/.snapshot
-   * @param rootDir hbase root directory
-   * @return the base directory in which all snapshots are kept
-   */
-  public static Path getSnapshotRootDir(final Path rootDir) {
-    return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
-  }
-
-  /**
-   * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root
-   * directory and all the data files for a snapshot are kept under this directory.
-   * @param snapshot snapshot being taken
-   * @param rootDir hbase root directory
-   * @return the final directory for the completed snapshot
-   */
-  public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) {
-    return getCompletedSnapshotDir(snapshot.getName(), rootDir);
-  }
-
-  /**
-   * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root
-   * directory and all the data files for a snapshot are kept under this directory.
-   * @param snapshotName name of the snapshot being taken
-   * @param rootDir hbase root directory
-   * @return the final directory for the completed snapshot
-   */
-  public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) {
-    return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName);
-  }
-
-  /**
-   * Get the general working directory for snapshots - where they are built, where they are
-   * temporarily copied on export, etc.
-   * @param rootDir root directory of the HBase installation
-   * @return Path to the snapshot tmp directory, relative to the passed root directory
-   */
-  public static Path getWorkingSnapshotDir(final Path rootDir) {
-    return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME);
-  }
-
-  /**
-   * Get the directory to build a snapshot, before it is finalized
-   * @param snapshot snapshot that will be built
-   * @param rootDir root directory of the hbase installation
-   * @return {@link Path} where one can build a snapshot
-   */
-  public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) {
-    return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName());
-  }
-
-  /**
-   * Get the directory to build a snapshot, before it is finalized
-   * @param snapshotName name of the snapshot
-   * @param rootDir root directory of the hbase installation
-   * @return {@link Path} where one can build a snapshot
-   */
-  public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) {
-    return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName);
-  }
-
-  /**
-   * Get the directory to store the snapshot instance
-   * @param snapshotsDir hbase-global directory for storing all snapshots
-   * @param snapshotName name of the snapshot to take
-   * @return the final directory for the completed snapshot
-   */
-  private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) {
-    return new Path(snapshotsDir, snapshotName);
-  }
-
-  /**
-   * @param rootDir hbase root directory
-   * @return the directory for all completed snapshots;
-   */
-  public static final Path getSnapshotsDir(Path rootDir) {
-    return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
-  }
-
-  /**
    * Convert the passed snapshot description into a 'full' snapshot description based on default
    * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't
    * supplied to their default values.
@@ -267,94 +118,6 @@ public final class SnapshotDescriptionUtils {
   }
 
   /**
-   * Write the snapshot description into the working directory of a snapshot
-   * @param snapshot description of the snapshot being taken
-   * @param workingDir working directory of the snapshot
-   * @param fs {@link FileSystem} on which the snapshot should be taken
-   * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on
-   *           failure
-   */
-  public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs)
-      throws IOException {
-    FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
-      HConstants.DATA_FILE_UMASK_KEY);
-    Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
-    try {
-      FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true);
-      try {
-        snapshot.writeTo(out);
-      } finally {
-        out.close();
-      }
-    } catch (IOException e) {
-      // if we get an exception, try to remove the snapshot info
-      if (!fs.delete(snapshotInfo, false)) {
-        String msg = "Couldn't delete snapshot info file: " + snapshotInfo;
-        LOG.error(msg);
-        throw new IOException(msg);
-      }
-    }
-  }
-
-  /**
-   * Create in-progress tag under .tmp of in-progress snapshot
-   * */
-  public static void createInProgressTag(Path workingDir, FileSystem fs) throws IOException {
-    FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(),
-      HConstants.DATA_FILE_UMASK_KEY);
-    Path snapshot_in_progress = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS);
-    FSUtils.create(fs, snapshot_in_progress, perms, true);
-  }
-
-  /**
-   * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
-   * @param fs filesystem where the snapshot was taken
-   * @param snapshotDir directory where the snapshot was stored
-   * @return the stored snapshot description
-   * @throws CorruptedSnapshotException if the
-   * snapshot cannot be read
-   */
-  public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir)
-      throws CorruptedSnapshotException {
-    Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
-    try {
-      FSDataInputStream in = null;
-      try {
-        in = fs.open(snapshotInfo);
-        SnapshotDescription desc = SnapshotDescription.parseFrom(in);
-        return desc;
-      } finally {
-        if (in != null) in.close();
-      }
-    } catch (IOException e) {
-      throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
-    }
-  }
-
-  /**
-   * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot
-   * as 'complete'.
-   * @param snapshot description of the snapshot being tabken
-   * @param rootdir root directory of the hbase installation
-   * @param workingDir directory where the in progress snapshot was built
-   * @param fs {@link FileSystem} where the snapshot was built
-   * @throws org.apache.hadoop.hbase.snapshot.SnapshotCreationException if the
-   * snapshot could not be moved
-   * @throws IOException the filesystem could not be reached
-   */
-  public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir,
-      FileSystem fs) throws SnapshotCreationException, IOException {
-    Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir);
-    LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
-        + finishedDir);
-    if (!fs.rename(workingDir, finishedDir)) {
-      throw new SnapshotCreationException(
-          "Failed to move working directory(" + workingDir + ") to completed directory("
-              + finishedDir + ").", ProtobufUtil.createSnapshotDesc(snapshot));
-    }
-  }
-
-  /**
    * Check if the user is this table snapshot's owner
    * @param snapshot the table snapshot description
    * @param user the user

http://git-wip-us.apache.org/repos/asf/hbase/blob/159a67c6/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index 6f7fd8f..b7fb50d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -36,14 +36,20 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.SnapshotDescription;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.fs.MasterStorage;
+import org.apache.hadoop.hbase.fs.StorageContext;
+import org.apache.hadoop.hbase.fs.StorageIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.LegacyPathIdentifier;
+import org.apache.hadoop.hbase.fs.legacy.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -53,7 +59,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.fs.legacy.io.HFileLink;
 import org.apache.hadoop.hbase.io.WALLink;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.util.FSUtils;
 
@@ -135,24 +140,21 @@ public final class SnapshotInfo extends Configured implements Tool {
 
     private final HBaseProtos.SnapshotDescription snapshot;
     private final TableName snapshotTable;
-    private final Configuration conf;
-    private final FileSystem fs;
+    private final MasterStorage<? extends StorageIdentifier> masterStorage;
 
-    SnapshotStats(final Configuration conf, final FileSystem fs,
+    SnapshotStats(final MasterStorage<? extends StorageIdentifier> masterStorage,
         final SnapshotDescription snapshot)
     {
       this.snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
       this.snapshotTable = TableName.valueOf(snapshot.getTable());
-      this.conf = conf;
-      this.fs = fs;
+      this.masterStorage = masterStorage;
     }
 
-    SnapshotStats(final Configuration conf, final FileSystem fs,
+    SnapshotStats(final MasterStorage<? extends StorageIdentifier> masterStorage,
         final HBaseProtos.SnapshotDescription snapshot) {
       this.snapshot = snapshot;
       this.snapshotTable = TableName.valueOf(snapshot.getTable());
-      this.conf = conf;
-      this.fs = fs;
+      this.masterStorage = masterStorage;
     }
 
 
@@ -225,7 +227,7 @@ public final class SnapshotInfo extends Configured implements Tool {
      *    with other snapshots and tables
      *
      *    This is only calculated when
-     *  {@link #getSnapshotStats(Configuration, HBaseProtos.SnapshotDescription, Map)}
+     *    {@link #getSnapshotStats(Configuration, HBaseProtos.SnapshotDescription, Map)}
      *    is called with a non-null Map
      */
     public long getNonSharedArchivedStoreFilesSize() {
@@ -265,7 +267,7 @@ public final class SnapshotInfo extends Configured implements Tool {
         Path parentDir = filePath.getParent();
         Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName());
         try {
-          if (FSUtils.listStatus(fs, backRefDir) == null) {
+          if (FSUtils.listStatus(masterStorage.getFileSystem(), backRefDir) == null) {
             return false;
           }
         } catch (IOException e) {
@@ -287,6 +289,8 @@ public final class SnapshotInfo extends Configured implements Tool {
     FileInfo addStoreFile(final HRegionInfo region, final String family,
         final SnapshotRegionManifest.StoreFile storeFile,
         final Map<Path, Integer> filesMap) throws IOException {
+      Configuration conf = masterStorage.getConfiguration();
+      FileSystem fs = masterStorage.getFileSystem();
       HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(),
               family, storeFile.getName());
       boolean isCorrupted = false;
@@ -328,10 +332,10 @@ public final class SnapshotInfo extends Configured implements Tool {
      * @return the log information
      */
     FileInfo addLogFile(final String server, final String logfile) throws IOException {
-      WALLink logLink = new WALLink(conf, server, logfile);
+      WALLink logLink = new WALLink(masterStorage.getConfiguration(), server, logfile);
       long size = -1;
       try {
-        size = logLink.getFileStatus(fs).getLen();
+        size = logLink.getFileStatus(masterStorage.getFileSystem()).getLen();
         logSize.addAndGet(size);
         logsCount.incrementAndGet();
       } catch (FileNotFoundException e) {
@@ -342,10 +346,10 @@ public final class SnapshotInfo extends Configured implements Tool {
   }
 
   private boolean printSizeInBytes = false;
-  private FileSystem fs;
-  private Path rootDir;
+  private MasterStorage<? extends StorageIdentifier> masterStorage;
 
-  private SnapshotManifest snapshotManifest;
+  private HBaseProtos.SnapshotDescription snapshotDesc;
+  private HTableDescriptor snapshotTable;
 
   @Override
   @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION",
@@ -391,11 +395,14 @@ public final class SnapshotInfo extends Configured implements Tool {
       }
     }
 
+    // instantiate MasterStorage
+    masterStorage = MasterStorage.open(conf, false);
+
     // List Available Snapshots
     if (listSnapshots) {
       SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
       System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME");
-      for (SnapshotDescription desc: getSnapshotList(conf)) {
+      for (SnapshotDescription desc: getSnapshotList(masterStorage)) {
         System.out.printf("%-20s | %20s | %s%n",
                           desc.getName(),
                           df.format(new Date(desc.getCreationTime())),
@@ -410,9 +417,8 @@ public final class SnapshotInfo extends Configured implements Tool {
       return 1;
     }
 
-    rootDir = FSUtils.getRootDir(conf);
-    fs = FileSystem.get(rootDir.toUri(), conf);
-    LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir);
+    LOG.debug("fs=" + masterStorage.getFileSystem().getUri().toString() + " root=" +
+        ((LegacyPathIdentifier)masterStorage.getRootContainer()).path);
 
     // Load snapshot information
     if (!loadSnapshotInfo(snapshotName)) {
@@ -433,15 +439,8 @@ public final class SnapshotInfo extends Configured implements Tool {
    * @return false if snapshot is not found
    */
   private boolean loadSnapshotInfo(final String snapshotName) throws IOException {
-    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
-    if (!fs.exists(snapshotDir)) {
-      LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir);
-      return false;
-    }
-
-    HBaseProtos.SnapshotDescription snapshotDesc =
-        SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
-    snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
+    snapshotDesc = masterStorage.getSnapshot(snapshotName);
+    snapshotTable = masterStorage.getTableDescriptorForSnapshot(snapshotDesc);
     return true;
   }
 
@@ -449,7 +448,6 @@ public final class SnapshotInfo extends Configured implements Tool {
    * Dump the {@link SnapshotDescription}
    */
   private void printInfo() {
-    HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
     System.out.println("Snapshot Info");
     System.out.println("----------------------------------------");
@@ -468,7 +466,7 @@ public final class SnapshotInfo extends Configured implements Tool {
   private void printSchema() {
     System.out.println("Table Descriptor");
     System.out.println("----------------------------------------");
-    System.out.println(snapshotManifest.getTableDescriptor().toString());
+    System.out.println(snapshotTable.toString());
     System.out.println();
   }
 
@@ -483,30 +481,30 @@ public final class SnapshotInfo extends Configured implements Tool {
     }
 
     // Collect information about hfiles and logs in the snapshot
-    final HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
     final String table = snapshotDesc.getTable();
     SnapshotDescription desc = new SnapshotDescription(snapshotDesc.getName(),
         snapshotDesc.getTable(), ProtobufUtil.createSnapshotType(snapshotDesc.getType()),
         snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
-    final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc);
-    SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
-        "SnapshotInfo",
-      new SnapshotReferenceUtil.SnapshotVisitor() {
-        @Override
-        public void storeFile(final HRegionInfo regionInfo, final String family,
-            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
-          if (storeFile.hasReference()) return;
-
-          SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile, null);
-          if (showFiles) {
-            String state = info.getStateToString();
-            System.out.printf("%8s %s/%s/%s/%s %s%n",
-              (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
-              table, regionInfo.getEncodedName(), family, storeFile.getName(),
-              state == null ? "" : "(" + state + ")");
+    final SnapshotStats stats = new SnapshotStats(masterStorage, desc);
+    // TODO: add concurrent API to MasterStorage
+    masterStorage.visitSnapshotStoreFiles(snapshotDesc, StorageContext.DATA,
+        new MasterStorage.SnapshotStoreFileVisitor() {
+          @Override
+          public void visitSnapshotStoreFile(HBaseProtos.SnapshotDescription snapshot,
+              StorageContext ctx, HRegionInfo hri, String familyName,
+              SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+            if (storeFile.hasReference()) return;
+
+            SnapshotStats.FileInfo info = stats.addStoreFile(hri, familyName, storeFile, null);
+            if (showFiles) {
+              String state = info.getStateToString();
+              System.out.printf("%8s %s/%s/%s/%s %s%n",
+                  (info.isMissing() ? "-" : fileSizeToString(info.getSize())),
+                  table, hri.getEncodedName(), familyName, storeFile.getName(),
+                  state == null ? "" : "(" + state + ")");
+            }
           }
-        }
-    });
+        });
 
     // Dump the stats
     System.out.println();
@@ -569,7 +567,8 @@ public final class SnapshotInfo extends Configured implements Tool {
     HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(
         snapshot);
 
-    return getSnapshotStats(conf, snapshotDesc, null);
+    MasterStorage<? extends StorageIdentifier> masterStorage = MasterStorage.open(conf, false);
+    return getSnapshotStats(masterStorage.getConfiguration(), snapshotDesc, null);
   }
 
   /**
@@ -579,53 +578,43 @@ public final class SnapshotInfo extends Configured implements Tool {
    * @param filesMap {@link Map} store files map for all snapshots, it may be null
    * @return the snapshot stats
    */
-  public static SnapshotStats getSnapshotStats(final Configuration conf,
-      final HBaseProtos.SnapshotDescription snapshotDesc,
+  public static SnapshotStats getSnapshotStats(
+      final Configuration conf, final HBaseProtos.SnapshotDescription snapshotDesc,
       final Map<Path, Integer> filesMap) throws IOException {
-    Path rootDir = FSUtils.getRootDir(conf);
-    FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
-    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
-    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
-    final SnapshotStats stats = new SnapshotStats(conf, fs, snapshotDesc);
-    SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
-        "SnapshotsStatsAggregation", new SnapshotReferenceUtil.SnapshotVisitor() {
+    MasterStorage<? extends StorageIdentifier> masterStorage = MasterStorage.open(conf, false);
+    final SnapshotStats stats = new SnapshotStats(masterStorage, snapshotDesc);
+    masterStorage.visitSnapshotStoreFiles(snapshotDesc, StorageContext.DATA,
+        new MasterStorage.SnapshotStoreFileVisitor() {
           @Override
-          public void storeFile(final HRegionInfo regionInfo, final String family,
-              final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+          public void visitSnapshotStoreFile(HBaseProtos.SnapshotDescription snapshot,
+              StorageContext ctx, HRegionInfo hri, String familyName,
+              SnapshotRegionManifest.StoreFile storeFile) throws IOException {
             if (!storeFile.hasReference()) {
-              stats.addStoreFile(regionInfo, family, storeFile, filesMap);
+              stats.addStoreFile(hri, familyName, storeFile, filesMap);
             }
-          }});
+          }
+        });
     return stats;
   }
 
   /**
    * Returns the list of available snapshots in the specified location
-   * @param conf the {@link Configuration} to use
+   * @param masterStorage the {@link MasterStorage} to use
    * @return the list of snapshots
    */
-  public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
-      throws IOException {
-    Path rootDir = FSUtils.getRootDir(conf);
-    FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
-    Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
-    FileStatus[] snapshots = fs.listStatus(snapshotDir,
-        new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
-    List<SnapshotDescription> snapshotLists =
-      new ArrayList<SnapshotDescription>(snapshots.length);
-    for (FileStatus snapshotDirStat: snapshots) {
-      HBaseProtos.SnapshotDescription snapshotDesc =
-          SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath());
-      snapshotLists.add(new SnapshotDescription(snapshotDesc.getName(),
-          snapshotDesc.getTable(), ProtobufUtil.createSnapshotType(snapshotDesc.getType()),
-          snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion()));
+  public static List<SnapshotDescription> getSnapshotList(
+      final MasterStorage<? extends StorageIdentifier> masterStorage) throws IOException {
+    List<HBaseProtos.SnapshotDescription> descriptions = masterStorage.getSnapshots();
+    List<SnapshotDescription> snapshotLists = new ArrayList(descriptions.size());
+    for (HBaseProtos.SnapshotDescription desc: masterStorage.getSnapshots()) {
+      snapshotLists.add(ProtobufUtil.createSnapshotDesc(desc));
     }
     return snapshotLists;
   }
 
   /**
    * Gets the store files map for snapshot
-   * @param conf the {@link Configuration} to use
+   * @param masterStorage the {@link MasterStorage} to use
    * @param snapshot {@link SnapshotDescription} to get stats from
    * @param exec the {@link ExecutorService} to use
    * @param filesMap {@link Map} the map to put the mapping entries
@@ -634,33 +623,32 @@ public final class SnapshotInfo extends Configured implements Tool {
    * @param uniqueHFilesMobSize {@link AtomicLong} the accumulated mob store file size shared
    * @return the snapshot stats
    */
-  private static void getSnapshotFilesMap(final Configuration conf,
+  private static void getSnapshotFilesMap(
+      final MasterStorage<? extends StorageIdentifier> masterStorage,
       final SnapshotDescription snapshot, final ExecutorService exec,
       final ConcurrentHashMap<Path, Integer> filesMap,
       final AtomicLong uniqueHFilesArchiveSize, final AtomicLong uniqueHFilesSize,
       final AtomicLong uniqueHFilesMobSize) throws IOException {
     HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(
         snapshot);
-    Path rootDir = FSUtils.getRootDir(conf);
-    final FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
-
-    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
-    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
-    SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, exec,
-        new SnapshotReferenceUtil.SnapshotVisitor() {
-          @Override public void storeFile(final HRegionInfo regionInfo, final String family,
-              final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+    masterStorage.visitSnapshotStoreFiles(snapshotDesc, StorageContext.DATA,
+        new MasterStorage.SnapshotStoreFileVisitor() {
+          @Override
+          public void visitSnapshotStoreFile(HBaseProtos.SnapshotDescription snapshot,
+              StorageContext ctx, HRegionInfo hri, String familyName,
+              SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+            Configuration conf = masterStorage.getConfiguration();
+            FileSystem fs = masterStorage.getFileSystem();
             if (!storeFile.hasReference()) {
-              HFileLink link = HFileLink
-                  .build(conf, TableName.valueOf(snapshot.getTable()), regionInfo.getEncodedName(),
-                      family, storeFile.getName());
+              HFileLink link = HFileLink.build(conf, TableName.valueOf(snapshot.getTable()),
+                  hri.getEncodedName(), familyName, storeFile.getName());
               long size;
               Integer count;
               Path p;
               AtomicLong al;
               int c = 0;
 
-              if (fs.exists(link.getArchivePath())) {
+              if (masterStorage.getFileSystem().exists(link.getArchivePath())) {
                 p = link.getArchivePath();
                 al = uniqueHFilesArchiveSize;
                 size = fs.getFileStatus(p).getLen();
@@ -685,7 +673,7 @@ public final class SnapshotInfo extends Configured implements Tool {
               filesMap.put(p, ++c);
             }
           }
-        });
+    });
   }
 
   /**
@@ -699,7 +687,8 @@ public final class SnapshotInfo extends Configured implements Tool {
   public static Map<Path, Integer> getSnapshotsFilesMap(final Configuration conf,
       AtomicLong uniqueHFilesArchiveSize, AtomicLong uniqueHFilesSize,
       AtomicLong uniqueHFilesMobSize) throws IOException {
-    List<SnapshotDescription> snapshotList = getSnapshotList(conf);
+    MasterStorage<? extends StorageIdentifier> masterStorage = MasterStorage.open(conf, false);
+    List<SnapshotDescription> snapshotList = getSnapshotList(masterStorage);
 
 
     if (snapshotList.size() == 0) {
@@ -712,7 +701,7 @@ public final class SnapshotInfo extends Configured implements Tool {
 
     try {
       for (final SnapshotDescription snapshot : snapshotList) {
-        getSnapshotFilesMap(conf, snapshot, exec, fileMap, uniqueHFilesArchiveSize,
+        getSnapshotFilesMap(masterStorage, snapshot, exec, fileMap, uniqueHFilesArchiveSize,
             uniqueHFilesSize, uniqueHFilesMobSize);
       }
     } finally {


Mime
View raw message