hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject svn commit: r1593139 [3/6] - in /hbase/trunk: hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ hbase-protocol/src/main/protobuf/ hbase-server/src/main/java/org/apache/hadoop/hbase/client/ hbase-server/src/main/java/org/apache/ha...
Date Wed, 07 May 2014 21:28:13 GMT
Modified: hbase/trunk/hbase-protocol/src/main/protobuf/MapReduce.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/protobuf/MapReduce.proto?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/protobuf/MapReduce.proto (original)
+++ hbase/trunk/hbase-protocol/src/main/protobuf/MapReduce.proto Wed May  7 21:28:12 2014
@@ -30,6 +30,7 @@ message ScanMetrics {
 }
 
 message TableSnapshotRegionSplit {
-  optional RegionSpecifier region = 1;
   repeated string locations = 2;
+  optional TableSchema table = 3;
+  optional RegionInfo region = 4;
 }

Added: hbase/trunk/hbase-protocol/src/main/protobuf/Snapshot.proto
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-protocol/src/main/protobuf/Snapshot.proto?rev=1593139&view=auto
==============================================================================
--- hbase/trunk/hbase-protocol/src/main/protobuf/Snapshot.proto (added)
+++ hbase/trunk/hbase-protocol/src/main/protobuf/Snapshot.proto Wed May  7 21:28:12 2014
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "SnapshotProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "FS.proto";
+import "HBase.proto";
+
+message SnapshotFileInfo {
+  enum Type {
+    HFILE = 1;
+    WAL = 2;
+  }
+
+  required Type type = 1;
+
+  optional string hfile = 3;
+
+  optional string wal_server = 4;
+  optional string wal_name = 5;
+}
+
+message SnapshotRegionManifest {
+  optional int32 version = 1;
+
+  required RegionInfo region_info = 2;
+  repeated FamilyFiles family_files = 3;
+
+  message StoreFile {
+    required string name = 1;
+    optional Reference reference = 2;
+
+    // TODO: Add checksums or other fields to verify the file
+    optional uint64 file_size = 3;
+  }
+
+  message FamilyFiles {
+    required bytes family_name = 1;
+    repeated StoreFile store_files = 2;
+  }
+}
+
+message SnapshotDataManifest {
+  required TableSchema table_schema = 1;
+  repeated SnapshotRegionManifest region_manifests = 2;
+}

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/TableSnapshotScanner.java Wed May  7 21:28:12 2014
@@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.client;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Set;
+import java.util.List;
 import java.util.UUID;
 
 import org.apache.commons.logging.Log;
@@ -32,16 +32,16 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.util.FSUtils;
 
 /**
  * A Scanner which performs a scan over snapshot files. Using this class requires copying the
@@ -99,8 +99,7 @@ public class TableSnapshotScanner extend
    */
   public TableSnapshotScanner(Configuration conf, Path restoreDir,
       String snapshotName, Scan scan) throws IOException {
-    this(conf, new Path(conf.get(HConstants.HBASE_DIR)),
-      restoreDir, snapshotName, scan);
+    this(conf, FSUtils.getRootDir(conf), restoreDir, snapshotName, scan);
   }
 
   /**
@@ -128,22 +127,21 @@ public class TableSnapshotScanner extend
 
   private void init() throws IOException {
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
+    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
 
-    //load table descriptor
-    htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+    // load table descriptor
+    htd = manifest.getTableDescriptor();
 
-    Set<String> snapshotRegionNames
-      = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
-    if (snapshotRegionNames == null) {
+    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+    if (regionManifests == null) {
       throw new IllegalArgumentException("Snapshot seems empty");
     }
 
-    regions = new ArrayList<HRegionInfo>(snapshotRegionNames.size());
-    for (String regionName : snapshotRegionNames) {
+    regions = new ArrayList<HRegionInfo>(regionManifests.size());
+    for (SnapshotRegionManifest regionManifest : regionManifests) {
       // load region descriptor
-      Path regionDir = new Path(snapshotDir, regionName);
-      HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs,
-          regionDir);
+      HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo());
 
       if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(),
           hri.getStartKey(), hri.getEndKey())) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java Wed May  7 21:28:12 2014
@@ -190,7 +190,7 @@ public class Reference {
     }
   }
 
-  FSProtos.Reference convert() {
+  public FSProtos.Reference convert() {
     FSProtos.Reference.Builder builder = FSProtos.Reference.newBuilder();
     builder.setRange(isTopFileRegion(getFileRegion())?
       FSProtos.Reference.Range.TOP: FSProtos.Reference.Range.BOTTOM);
@@ -198,7 +198,7 @@ public class Reference {
     return builder.build();
   }
 
-  static Reference convert(final FSProtos.Reference r) {
+  public static Reference convert(final FSProtos.Reference r) {
     Reference result = new Reference();
     result.splitkey = r.getSplitkey().toByteArray();
     result.region = r.getRange() == FSProtos.Reference.Range.TOP? Range.top: Range.bottom;

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java Wed May  7 21:28:12 2014
@@ -25,10 +25,8 @@ import java.io.IOException;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.UUID;
 
-import com.google.protobuf.HBaseZeroCopyByteString;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -37,7 +35,6 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
 import org.apache.hadoop.hbase.HDFSBlocksDistribution.HostAndWeight;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -49,18 +46,15 @@ import org.apache.hadoop.hbase.client.Sc
 import org.apache.hadoop.hbase.client.TableSnapshotScanner;
 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -82,7 +76,7 @@ import com.google.common.annotations.Vis
  * while there are jobs reading from snapshot files.
  * <p>
  * Usage is similar to TableInputFormat, and
- * {@link TableMapReduceUtil#initTableSnapshotMapperJob(String, Scan, Class, Class, Class, Job, 
+ * {@link TableMapReduceUtil#initTableSnapshotMapperJob(String, Scan, Class, Class, Class, Job,
  *   boolean, Path)}
  * can be used to configure the job.
  * <pre>{@code
@@ -101,12 +95,12 @@ import com.google.common.annotations.Vis
  * <p>
  * HBase owns all the data and snapshot files on the filesystem. Only the HBase user can read from
  * snapshot files and data files. HBase also enforces security because all the requests are handled
- * by the server layer, and the user cannot read from the data files directly. 
- * To read from snapshot files directly from the file system, the user who is running the MR job 
- * must have sufficient permissions to access snapshot and reference files. 
- * This means that to run mapreduce over snapshot files, the MR job has to be run as the HBase 
- * user or the user must have group or other priviledges in the filesystem (See HBASE-8369). 
- * Note that, given other users access to read from snapshot/data files will completely circumvent 
+ * by the server layer, and the user cannot read from the data files directly.
+ * To read from snapshot files directly from the file system, the user who is running the MR job
+ * must have sufficient permissions to access snapshot and reference files.
+ * This means that to run mapreduce over snapshot files, the MR job has to be run as the HBase
+ * user or the user must have group or other priviledges in the filesystem (See HBASE-8369).
+ * Note that, given other users access to read from snapshot/data files will completely circumvent
  * the access control enforced by HBase.
  * @see TableSnapshotScanner
  */
@@ -119,22 +113,25 @@ public class TableSnapshotInputFormat ex
   private static final Log LOG = LogFactory.getLog(TableSnapshotInputFormat.class);
 
   /** See {@link #getBestLocations(Configuration, HDFSBlocksDistribution)} */
-  private static final String LOCALITY_CUTOFF_MULTIPLIER = 
+  private static final String LOCALITY_CUTOFF_MULTIPLIER =
       "hbase.tablesnapshotinputformat.locality.cutoff.multiplier";
   private static final float DEFAULT_LOCALITY_CUTOFF_MULTIPLIER = 0.8f;
 
   private static final String SNAPSHOT_NAME_KEY = "hbase.TableSnapshotInputFormat.snapshot.name";
   private static final String TABLE_DIR_KEY = "hbase.TableSnapshotInputFormat.table.dir";
 
-  public static class TableSnapshotRegionSplit extends InputSplit implements Writable {
-    private String regionName;
+  @VisibleForTesting
+  static class TableSnapshotRegionSplit extends InputSplit implements Writable {
+    private HTableDescriptor htd;
+    private HRegionInfo regionInfo;
     private String[] locations;
 
     // constructor for mapreduce framework / Writable
     public TableSnapshotRegionSplit() { }
 
-    TableSnapshotRegionSplit(String regionName, List<String> locations) {
-      this.regionName = regionName;
+    TableSnapshotRegionSplit(HTableDescriptor htd, HRegionInfo regionInfo, List<String> locations) {
+      this.htd = htd;
+      this.regionInfo = regionInfo;
       if (locations == null || locations.isEmpty()) {
         this.locations = new String[0];
       } else {
@@ -158,9 +155,8 @@ public class TableSnapshotInputFormat ex
     public void write(DataOutput out) throws IOException {
     MapReduceProtos.TableSnapshotRegionSplit.Builder builder =
       MapReduceProtos.TableSnapshotRegionSplit.newBuilder()
-        .setRegion(RegionSpecifier.newBuilder()
-          .setType(RegionSpecifierType.ENCODED_REGION_NAME)
-          .setValue(HBaseZeroCopyByteString.wrap(Bytes.toBytes(regionName))).build());
+        .setTable(htd.convert())
+        .setRegion(HRegionInfo.convert(regionInfo));
 
       for (String location : locations) {
         builder.addLocations(location);
@@ -180,16 +176,17 @@ public class TableSnapshotInputFormat ex
       int len = in.readInt();
       byte[] buf = new byte[len];
       in.readFully(buf);
-      MapReduceProtos.TableSnapshotRegionSplit split = 
+      MapReduceProtos.TableSnapshotRegionSplit split =
           MapReduceProtos.TableSnapshotRegionSplit.PARSER.parseFrom(buf);
-      this.regionName = Bytes.toString(split.getRegion().getValue().toByteArray());
+      this.htd = HTableDescriptor.convert(split.getTable());
+      this.regionInfo = HRegionInfo.convert(split.getRegion());
       List<String> locationsList = split.getLocationsList();
       this.locations = locationsList.toArray(new String[locationsList.size()]);
     }
   }
 
   @VisibleForTesting
-  static class TableSnapshotRegionRecordReader extends 
+  static class TableSnapshotRegionRecordReader extends
     RecordReader<ImmutableBytesWritable, Result> {
     private TableSnapshotRegionSplit split;
     private Scan scan;
@@ -205,23 +202,13 @@ public class TableSnapshotInputFormat ex
 
       Configuration conf = context.getConfiguration();
       this.split = (TableSnapshotRegionSplit) split;
-      String regionName = this.split.regionName;
-      String snapshotName = getSnapshotName(conf);
-      Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
-      FileSystem fs = rootDir.getFileSystem(conf);
+      HTableDescriptor htd = this.split.htd;
+      HRegionInfo hri = this.split.regionInfo;
+      FileSystem fs = FSUtils.getCurrentFileSystem(conf);
 
       Path tmpRootDir = new Path(conf.get(TABLE_DIR_KEY)); // This is the user specified root
       // directory where snapshot was restored
 
-      Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
-
-      //load table descriptor
-      HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
-
-      //load region descriptor
-      Path regionDir = new Path(snapshotDir, regionName);
-      HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
-
       // create scan
       String scanStr = conf.get(TableInputFormat.SCAN);
       if (scanStr == null) {
@@ -294,31 +281,28 @@ public class TableSnapshotInputFormat ex
     Configuration conf = job.getConfiguration();
     String snapshotName = getSnapshotName(conf);
 
-    Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+    Path rootDir = FSUtils.getRootDir(conf);
     FileSystem fs = rootDir.getFileSystem(conf);
 
     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
-
-    Set<String> snapshotRegionNames
-      = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
-    if (snapshotRegionNames == null) {
+    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
+    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+    if (regionManifests == null) {
       throw new IllegalArgumentException("Snapshot seems empty");
     }
 
     // load table descriptor
-    HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs,
-        snapshotDir);
+    HTableDescriptor htd = manifest.getTableDescriptor();
 
     Scan scan = TableMapReduceUtil.convertStringToScan(conf
       .get(TableInputFormat.SCAN));
     Path tableDir = new Path(conf.get(TABLE_DIR_KEY));
 
     List<InputSplit> splits = new ArrayList<InputSplit>();
-    for (String regionName : snapshotRegionNames) {
+    for (SnapshotRegionManifest regionManifest : regionManifests) {
       // load region descriptor
-      Path regionDir = new Path(snapshotDir, regionName);
-      HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs,
-          regionDir);
+      HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo());
 
       if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(),
           hri.getStartKey(), hri.getEndKey())) {
@@ -329,7 +313,7 @@ public class TableSnapshotInputFormat ex
 
         int len = Math.min(3, hosts.size());
         hosts = hosts.subList(0, len);
-        splits.add(new TableSnapshotRegionSplit(regionName, hosts));
+        splits.add(new TableSnapshotRegionSplit(htd, hri, hosts));
       }
     }
 
@@ -341,7 +325,7 @@ public class TableSnapshotInputFormat ex
    * weights into account, thus will treat every location passed from the input split as equal. We
    * do not want to blindly pass all the locations, since we are creating one split per region, and
    * the region's blocks are all distributed throughout the cluster unless favorite node assignment
-   * is used. On the expected stable case, only one location will contain most of the blocks as 
+   * is used. On the expected stable case, only one location will contain most of the blocks as
    * local.
    * On the other hand, in favored node assignment, 3 nodes will contain highly local blocks. Here
    * we are doing a simple heuristic, where we will pass all hosts which have at least 80%
@@ -391,7 +375,7 @@ public class TableSnapshotInputFormat ex
     Configuration conf = job.getConfiguration();
     conf.set(SNAPSHOT_NAME_KEY, snapshotName);
 
-    Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+    Path rootDir = FSUtils.getRootDir(conf);
     FileSystem fs = rootDir.getFileSystem(conf);
 
     restoreDir = new Path(restoreDir, UUID.randomUUID().toString());

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java Wed May  7 21:28:12 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.snapshot.
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
 
 import com.google.common.base.Preconditions;
 
@@ -107,8 +108,9 @@ public class CloneSnapshotHandler extend
     try {
       // 1. Execute the on-disk Clone
       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
       RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(conf, fs,
-          snapshot, snapshotDir, hTableDescriptor, tableRootDir, monitor, status);
+          manifest, hTableDescriptor, tableRootDir, monitor, status);
       metaChanges = restoreHelper.restoreHdfsRegions();
 
       // Clone operation should not have stuff to restore or remove

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java Wed May  7 21:28:12 2014
@@ -24,23 +24,18 @@ import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionListener;
 import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector;
 import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.MetricsMaster;
-import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.zookeeper.KeeperException;
 
@@ -64,7 +59,7 @@ public class DisabledTableSnapshotHandle
     super(snapshot, masterServices);
 
     // setup the timer
-    timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
+    timeoutInjector = getMasterTimerAndBindToMonitor(snapshot, conf, monitor);
   }
 
   @Override
@@ -80,8 +75,6 @@ public class DisabledTableSnapshotHandle
     try {
       timeoutInjector.start();
 
-      Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
-
       // 1. get all the regions hosting this table.
 
       // extract each pair to separate lists
@@ -95,34 +88,38 @@ public class DisabledTableSnapshotHandle
           + ClientSnapshotDescriptionUtils.toString(snapshot);
       LOG.info(msg);
       status.setStatus(msg);
-      for (HRegionInfo regionInfo : regions) {
+      for (HRegionInfo regionInfo: regions) {
         snapshotDisabledRegion(regionInfo);
       }
-
-      // 3. write the table info to disk
-      LOG.info("Starting to copy tableinfo for offline snapshot: " +
-      ClientSnapshotDescriptionUtils.toString(snapshot));
-      TableInfoCopyTask tableInfoCopyTask = new TableInfoCopyTask(this.monitor, snapshot, fs,
-          FSUtils.getRootDir(conf));
-      tableInfoCopyTask.call();
-      monitor.rethrowException();
-      status.setStatus("Finished copying tableinfo for snapshot of table: " +
-          snapshotTable);
     } catch (Exception e) {
       // make sure we capture the exception to propagate back to the client later
       String reason = "Failed snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot)
           + " due to exception:" + e.getMessage();
       ForeignException ee = new ForeignException(reason, e);
       monitor.receive(ee);
-      status.abort("Snapshot of table: "+ snapshotTable +
-          " failed because " + e.getMessage());
+      status.abort("Snapshot of table: "+ snapshotTable + " failed because " + e.getMessage());
     } finally {
       LOG.debug("Marking snapshot" + ClientSnapshotDescriptionUtils.toString(snapshot)
           + " as finished.");
 
-      // 6. mark the timer as finished - even if we got an exception, we don't need to time the
+      // 3. mark the timer as finished - even if we got an exception, we don't need to time the
       // operation any further
       timeoutInjector.complete();
     }
   }
+
+
+  /**
+   * Create a snapshot timer for the master which notifies the monitor when an error occurs
+   * @param snapshot snapshot to monitor
+   * @param conf configuration to use when getting the max snapshot life
+   * @param monitor monitor to notify when the snapshot life expires
+   * @return the timer to use update to signal the start and end of the snapshot
+   */
+  private TimeoutExceptionInjector getMasterTimerAndBindToMonitor(SnapshotDescription snapshot,
+      Configuration conf, ForeignExceptionListener monitor) {
+    long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(),
+      SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
+    return new TimeoutExceptionInjector(monitor, maxTime);
+  }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java Wed May  7 21:28:12 2014
@@ -25,17 +25,14 @@ import java.util.Set;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.errorhandling.ForeignException;
 import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.MetricsMaster;
 import org.apache.hadoop.hbase.procedure.Procedure;
 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.util.Pair;
 
 import com.google.common.collect.Lists;
@@ -99,14 +96,11 @@ public class EnabledTableSnapshotHandler
       LOG.info("Done waiting - online snapshot for " + this.snapshot.getName());
 
       // Take the offline regions as disabled
-      Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
       for (Pair<HRegionInfo, ServerName> region : regions) {
         HRegionInfo regionInfo = region.getFirst();
         if (regionInfo.isOffline() && (regionInfo.isSplit() || regionInfo.isSplitParent())) {
-          if (!fs.exists(new Path(snapshotDir, regionInfo.getEncodedName()))) {
-            LOG.info("Take disabled snapshot of offline region=" + regionInfo);
-            snapshotDisabledRegion(regionInfo);
-          }
+          LOG.info("Take disabled snapshot of offline region=" + regionInfo);
+          snapshotDisabledRegion(regionInfo);
         }
       }
     } catch (InterruptedException e) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java Wed May  7 21:28:12 2014
@@ -19,35 +19,28 @@ package org.apache.hadoop.hbase.master.s
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.catalog.MetaReader;
-import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
-import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.FSVisitor;
-import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 
 /**
  * General snapshot verification on the master.
@@ -110,14 +103,16 @@ public final class MasterSnapshotVerifie
    */
   public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers)
       throws CorruptedSnapshotException, IOException {
+    SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), fs,
+                                                      snapshotDir, snapshot);
     // verify snapshot info matches
     verifySnapshotDescription(snapshotDir);
 
     // check that tableinfo is a valid table description
-    verifyTableInfo(snapshotDir);
+    verifyTableInfo(manifest);
 
     // check that each region is valid
-    verifyRegions(snapshotDir);
+    verifyRegions(manifest);
   }
 
   /**
@@ -136,8 +131,16 @@ public final class MasterSnapshotVerifie
    * Check that the table descriptor for the snapshot is a valid table descriptor
    * @param snapshotDir snapshot directory to check
    */
-  private void verifyTableInfo(Path snapshotDir) throws IOException {
-    FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+  private void verifyTableInfo(final SnapshotManifest manifest) throws IOException {
+    HTableDescriptor htd = manifest.getTableDescriptor();
+    if (htd == null) {
+      throw new CorruptedSnapshotException("Missing Table Descriptor", snapshot);
+    }
+
+    if (!htd.getNameAsString().equals(snapshot.getTable())) {
+      throw new CorruptedSnapshotException("Invalid Table Descriptor. Expected "
+        + snapshot.getTable() + " name, got " + htd.getNameAsString(), snapshot);
+    }
   }
 
   /**
@@ -145,34 +148,36 @@ public final class MasterSnapshotVerifie
    * @param snapshotDir snapshot directory to check
    * @throws IOException if we can't reach hbase:meta or read the files from the FS
    */
-  private void verifyRegions(Path snapshotDir) throws IOException {
+  private void verifyRegions(final SnapshotManifest manifest) throws IOException {
     List<HRegionInfo> regions = MetaReader.getTableRegions(this.services.getCatalogTracker(),
         tableName);
 
-    Set<String> snapshotRegions = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
-    if (snapshotRegions == null) {
+    Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
+    if (regionManifests == null) {
       String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
       LOG.error(msg);
       throw new CorruptedSnapshotException(msg);
     }
 
     String errorMsg = "";
-    if (snapshotRegions.size() != regions.size()) {
-      errorMsg = "Regions moved during the snapshot '" + 
+    if (regionManifests.size() != regions.size()) {
+      errorMsg = "Regions moved during the snapshot '" +
                    ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
-                   regions.size() + " snapshotted=" + snapshotRegions.size() + ".";
+                   regions.size() + " snapshotted=" + regionManifests.size() + ".";
       LOG.error(errorMsg);
     }
 
     for (HRegionInfo region : regions) {
-      if (!snapshotRegions.contains(region.getEncodedName())) {
+      SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName());
+      if (regionManifest == null) {
         // could happen due to a move or split race.
         String mesg = " No snapshot region directory found for region:" + region;
         if (errorMsg.isEmpty()) errorMsg = mesg;
         LOG.error(mesg);
+        continue;
       }
 
-      verifyRegion(fs, snapshotDir, region);
+      verifyRegion(fs, manifest.getSnapshotDir(), region, regionManifest);
     }
     if (!errorMsg.isEmpty()) {
       throw new CorruptedSnapshotException(errorMsg);
@@ -185,65 +190,24 @@ public final class MasterSnapshotVerifie
    * @param snapshotDir snapshot directory to check
    * @param region the region to check
    */
-  private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region)
-      throws IOException {
-    // make sure we have region in the snapshot
-    Path regionDir = new Path(snapshotDir, region.getEncodedName());
-
-    // make sure we have the region info in the snapshot
-    Path regionInfo = new Path(regionDir, HRegionFileSystem.REGION_INFO_FILE);
-    // make sure the file exists
-    if (!fs.exists(regionInfo)) {
-      throw new CorruptedSnapshotException("No region info found for region:" + region, snapshot);
-    }
-
-    HRegionInfo found = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
-    if (!region.equals(found)) {
-      throw new CorruptedSnapshotException("Found region info (" + found
-        + ") doesn't match expected region:" + region, snapshot);
-    }
-
-    // make sure we have the expected recovered edits files
-    TakeSnapshotUtils.verifyRecoveredEdits(fs, snapshotDir, found, snapshot);
-
-     // make sure we have all the expected store files
-    SnapshotReferenceUtil.visitRegionStoreFiles(fs, regionDir, new FSVisitor.StoreFileVisitor() {
-      public void storeFile(final String regionNameSuffix, final String family,
-          final String hfileName) throws IOException {
-        verifyStoreFile(snapshotDir, region, family, hfileName);
+  private void verifyRegion(final FileSystem fs, final Path snapshotDir, final HRegionInfo region,
+      final SnapshotRegionManifest manifest) throws IOException {
+    HRegionInfo manifestRegionInfo = HRegionInfo.convert(manifest.getRegionInfo());
+    if (!region.equals(manifestRegionInfo)) {
+      String msg = "Manifest region info " + manifestRegionInfo +
+                   "doesn't match expected region:" + region;
+      throw new CorruptedSnapshotException(msg, snapshot);
+    }
+
+    // make sure we have all the expected store files
+    SnapshotReferenceUtil.visitRegionStoreFiles(manifest,
+        new SnapshotReferenceUtil.StoreFileVisitor() {
+      @Override
+      public void storeFile(final HRegionInfo regionInfo, final String family,
+          final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+        SnapshotReferenceUtil.verifyStoreFile(services.getConfiguration(), fs, snapshotDir,
+          snapshot, region, family, storeFile);
       }
     });
   }
-
-  private void verifyStoreFile(final Path snapshotDir, final HRegionInfo regionInfo,
-      final String family, final String fileName) throws IOException {
-    Path refPath = null;
-    if (StoreFileInfo.isReference(fileName)) {
-      // If is a reference file check if the parent file is present in the snapshot
-      Path snapshotHFilePath = new Path(new Path(
-          new Path(snapshotDir, regionInfo.getEncodedName()), family), fileName);
-      refPath = StoreFileInfo.getReferredToFile(snapshotHFilePath);
-      if (!fs.exists(refPath)) {
-        throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName, snapshot);
-      }
-    }
-
-    Path linkPath;
-    if (refPath != null && HFileLink.isHFileLink(refPath)) {
-      linkPath = new Path(family, refPath.getName());
-    } else if (HFileLink.isHFileLink(fileName)) {
-      linkPath = new Path(family, fileName);
-    } else {
-      linkPath = new Path(family, HFileLink.createHFileLinkName(tableName,
-        regionInfo.getEncodedName(), fileName));
-    }
-
-    // check if the linked file exists (in the archive, or in the table dir)
-    HFileLink link = new HFileLink(services.getConfiguration(), linkPath);
-    if (!link.exists(fs)) {
-      throw new CorruptedSnapshotException("Can't find hfile: " + fileName
-          + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath()
-          + ") directory for the primary table.", snapshot);
-    }
-  }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java Wed May  7 21:28:12 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.snapshot.
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
 
 /**
  * Handler to Restore a snapshot.
@@ -120,9 +121,11 @@ public class RestoreSnapshotHandler exte
       // 2. Execute the on-disk Restore
       LOG.debug("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
       Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
+      SnapshotManifest manifest = SnapshotManifest.open(masterServices.getConfiguration(), fs,
+                                                        snapshotDir, snapshot);
       RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
-          masterServices.getConfiguration(), fs,
-          snapshot, snapshotDir, hTableDescriptor, rootDir, monitor, status);
+          masterServices.getConfiguration(), fs, manifest,
+          this.hTableDescriptor, rootDir, monitor, status);
       RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
 
       // 3. Forces all the RegionStates to be offline

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java Wed May  7 21:28:12 2014
@@ -174,19 +174,24 @@ public class SnapshotFileCache implement
   // is an illegal access to the cache. Really we could do a mutex-guarded pointer swap on the
   // cache, but that seems overkill at the moment and isn't necessarily a bottleneck.
   public synchronized boolean contains(String fileName) throws IOException {
-    if (this.cache.contains(fileName)) return true;
-
-    refreshCache();
-
-    // then check again
-    return this.cache.contains(fileName);
+    boolean hasFile = this.cache.contains(fileName);
+    if (!hasFile) {
+      refreshCache();
+      // then check again
+      hasFile = this.cache.contains(fileName);
+    }
+    return hasFile;
   }
 
   private synchronized void refreshCache() throws IOException {
-    // get the status of the snapshots directory and <snapshot dir>/.tmp
-    FileStatus dirStatus, tempStatus;
+    long lastTimestamp = Long.MAX_VALUE;
+    boolean hasChanges = false;
+
+    // get the status of the snapshots directory and check if it is has changes
     try {
-      dirStatus = fs.getFileStatus(snapshotDir);
+      FileStatus dirStatus = fs.getFileStatus(snapshotDir);
+      lastTimestamp = dirStatus.getModificationTime();
+      hasChanges |= (lastTimestamp >= lastModifiedTime);
     } catch (FileNotFoundException e) {
       if (this.cache.size() > 0) {
         LOG.error("Snapshot directory: " + snapshotDir + " doesn't exist");
@@ -194,16 +199,28 @@ public class SnapshotFileCache implement
       return;
     }
 
+    // get the status of the snapshots temporary directory and check if it has changes
+    // The top-level directory timestamp is not updated, so we have to check the inner-level.
     try {
       Path snapshotTmpDir = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOT_TMP_DIR_NAME);
-      tempStatus = fs.getFileStatus(snapshotTmpDir);
+      FileStatus tempDirStatus = fs.getFileStatus(snapshotTmpDir);
+      lastTimestamp = Math.min(lastTimestamp, tempDirStatus.getModificationTime());
+      hasChanges |= (lastTimestamp >= lastModifiedTime);
+      if (!hasChanges) {
+        FileStatus[] tmpSnapshots = FSUtils.listStatus(fs, snapshotDir);
+        if (tmpSnapshots != null) {
+          for (FileStatus dirStatus: tmpSnapshots) {
+            lastTimestamp = Math.min(lastTimestamp, dirStatus.getModificationTime());
+          }
+          hasChanges |= (lastTimestamp >= lastModifiedTime);
+        }
+      }
     } catch (FileNotFoundException e) {
-      tempStatus = dirStatus;
+      // Nothing todo, if the tmp dir is empty
     }
 
     // if the snapshot directory wasn't modified since we last check, we are done
-    if (dirStatus.getModificationTime() <= lastModifiedTime &&
-        tempStatus.getModificationTime() <= lastModifiedTime) {
+    if (!hasChanges) {
       return;
     }
 
@@ -213,8 +230,7 @@ public class SnapshotFileCache implement
     // However, snapshot directories are only created once, so this isn't an issue.
 
     // 1. update the modified time
-    this.lastModifiedTime = Math.min(dirStatus.getModificationTime(),
-                                     tempStatus.getModificationTime());
+    this.lastModifiedTime = lastTimestamp;
 
     // 2.clear the cache
     this.cache.clear();

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotHFileCleaner.java Wed May  7 21:28:12 2014
@@ -66,7 +66,7 @@ public class SnapshotHFileCleaner extend
   }
 
   @Override
-  public void setConf(Configuration conf) {
+  public void setConf(final Configuration conf) {
     super.setConf(conf);
     try {
       long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
@@ -77,7 +77,7 @@ public class SnapshotHFileCleaner extend
           "snapshot-hfile-cleaner-cache-refresher", new SnapshotFileCache.SnapshotFileInspector() {
             public Collection<String> filesUnderSnapshot(final Path snapshotDir)
                 throws IOException {
-              return SnapshotReferenceUtil.getHFileNames(fs, snapshotDir);
+              return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
             }
           });
     } catch (IOException e) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java Wed May  7 21:28:12 2014
@@ -70,11 +70,11 @@ import org.apache.hadoop.hbase.snapshot.
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.zookeeper.KeeperException;
 
@@ -540,9 +540,12 @@ public class SnapshotManager extends Mas
           + "' doesn't exist, can't take snapshot.", snapshot);
     }
 
-    // set the snapshot version, now that we are ready to take it
-    snapshot = snapshot.toBuilder().setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
-        .build();
+    // if not specified, set the snapshot format
+    if (!snapshot.hasVersion()) {
+      snapshot = snapshot.toBuilder()
+          .setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION)
+          .build();
+    }
 
     // call pre coproc hook
     MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
@@ -676,15 +679,16 @@ public class SnapshotManager extends Mas
 
     // read snapshot information
     SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
-    HTableDescriptor snapshotTableDesc =
-        FSTableDescriptors.getTableDescriptorFromFs(fs, snapshotDir);
+    SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
+        snapshotDir, fsSnapshot);
+    HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
     TableName tableName = TableName.valueOf(reqSnapshot.getTable());
 
     // stop tracking "abandoned" handlers
     cleanupSentinels();
 
     // Verify snapshot validity
-    SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, snapshotDir, fsSnapshot);
+    SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
 
     // Execute the restore/clone operation
     if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java Wed May  7 21:28:12 2014
@@ -48,14 +48,11 @@ import org.apache.hadoop.hbase.master.Ta
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.CopyRecoveredEditsTask;
-import org.apache.hadoop.hbase.snapshot.ReferenceRegionHFilesTask;
 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.snapshot.TableInfoCopyTask;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.zookeeper.KeeperException;
 
@@ -88,6 +85,9 @@ public abstract class TakeSnapshotHandle
   protected final TableLock tableLock;
   protected final MonitoredTask status;
   protected final TableName snapshotTable;
+  protected final SnapshotManifest snapshotManifest;
+
+  protected HTableDescriptor htd;
 
   /**
    * @param snapshot descriptor of the snapshot to take
@@ -107,6 +107,7 @@ public abstract class TakeSnapshotHandle
     this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
     this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
     this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
+    this.snapshotManifest = SnapshotManifest.create(conf, fs, workingDir, snapshot, monitor);
 
     this.tableLockManager = master.getTableLockManager();
     this.tableLock = this.tableLockManager.writeLock(
@@ -136,7 +137,7 @@ public abstract class TakeSnapshotHandle
                               // case of exceptions
     boolean success = false;
     try {
-      loadTableDescriptor(); // check that .tableinfo is present
+      this.htd = loadTableDescriptor(); // check that .tableinfo is present
       success = true;
     } finally {
       if (!success) {
@@ -162,8 +163,8 @@ public abstract class TakeSnapshotHandle
       // an external exception that gets captured here.
 
       // write down the snapshot info in the working directory
-      SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, this.fs);
-      new TableInfoCopyTask(monitor, snapshot, fs, rootDir).call();
+      SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
+      snapshotManifest.addTableDescriptor(this.htd);
       monitor.rethrowException();
 
       List<Pair<HRegionInfo, ServerName>> regionsAndLocations =
@@ -184,16 +185,19 @@ public abstract class TakeSnapshotHandle
         }
       }
 
+      // flush the in-memory state, and write the single manifest
+      status.setStatus("Consolidate snapshot: " + snapshot.getName());
+      snapshotManifest.consolidate();
+
       // verify the snapshot is valid
       status.setStatus("Verifying snapshot: " + snapshot.getName());
       verifier.verifySnapshot(this.workingDir, serverNames);
 
       // complete the snapshot, atomically moving from tmp to .snapshot dir.
       completeSnapshot(this.snapshotDir, this.workingDir, this.fs);
-      status.markComplete("Snapshot " + snapshot.getName() + " of table " + snapshotTable
-          + " completed");
-      LOG.info("Snapshot " + snapshot.getName() + " of table " + snapshotTable
-          + " completed");
+      msg = "Snapshot " + snapshot.getName() + " of table " + snapshotTable + " completed";
+      status.markComplete(msg);
+      LOG.info(msg);
       metricsSnapshot.addSnapshot(status.getCompletionTimestamp() - status.getStartTime());
     } catch (Exception e) {
       status.abort("Failed to complete snapshot " + snapshot.getName() + " on table " +
@@ -204,8 +208,7 @@ public abstract class TakeSnapshotHandle
       ForeignException ee = new ForeignException(reason, e);
       monitor.receive(ee);
       // need to mark this completed to close off and allow cleanup to happen.
-      cancel("Failed to take snapshot '" + ClientSnapshotDescriptionUtils.toString(snapshot)
-          + "' due to exception");
+      cancel(reason);
     } finally {
       LOG.debug("Launching cleanup of working dir:" + workingDir);
       try {
@@ -262,26 +265,10 @@ public abstract class TakeSnapshotHandle
    */
   protected void snapshotDisabledRegion(final HRegionInfo regionInfo)
       throws IOException {
-    // 2 copy the regionInfo files to the snapshot
-    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
-      workingDir, regionInfo);
-
-    // check for error for each region
-    monitor.rethrowException();
-
-    // 2 for each region, copy over its recovered.edits directory
-    Path regionDir = HRegion.getRegionDir(rootDir, regionInfo);
-    Path snapshotRegionDir = regionFs.getRegionDir();
-    new CopyRecoveredEditsTask(snapshot, monitor, fs, regionDir, snapshotRegionDir).call();
-    monitor.rethrowException();
-    status.setStatus("Completed copying recovered edits for offline snapshot of table: "
-        + snapshotTable);
-
-    // 2 reference all the files in the region
-    new ReferenceRegionHFilesTask(snapshot, monitor, regionDir, fs, snapshotRegionDir).call();
+    snapshotManifest.addRegion(FSUtils.getTableDir(rootDir, snapshotTable), regionInfo);
     monitor.rethrowException();
-    status.setStatus("Completed referencing HFiles for offline snapshot of table: " +
-        snapshotTable);
+    status.setStatus("Completed referencing HFiles for offline region " + regionInfo.toString() +
+        " of table: " + snapshotTable);
   }
 
   @Override

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureManager.java Wed May  7 21:28:12 2014
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.procedur
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
 
 @InterfaceAudience.Private
 @InterfaceStability.Evolving

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed May  7 21:28:12 2014
@@ -127,6 +127,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -193,7 +194,7 @@ public class HRegion implements HeapSize
 
   public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY =
       "hbase.hregion.scan.loadColumnFamiliesOnDemand";
-      
+
   /**
    * This is the global default value for durability. All tables/mutations not
    * defining a durability or using USE_DEFAULT will default to this value.
@@ -472,7 +473,7 @@ public class HRegion implements HeapSize
   private RegionServerAccounting rsAccounting;
   private List<Pair<Long, Long>> recentFlushes = new ArrayList<Pair<Long,Long>>();
   private long flushCheckInterval;
-  // flushPerChanges is to prevent too many changes in memstore    
+  // flushPerChanges is to prevent too many changes in memstore
   private long flushPerChanges;
   private long blockingMemStoreSize;
   final long threadWakeFrequency;
@@ -572,7 +573,7 @@ public class HRegion implements HeapSize
       throw new IllegalArgumentException(MEMSTORE_FLUSH_PER_CHANGES + " can not exceed "
           + MAX_FLUSH_PER_CHANGES);
     }
-    
+
     this.rowLockWaitDuration = conf.getInt("hbase.rowlock.wait.duration",
                     DEFAULT_ROWLOCK_WAIT_DURATION);
 
@@ -807,7 +808,7 @@ public class HRegion implements HeapSize
           for (Store store : this.stores.values()) {
             try {
               store.close();
-            } catch (IOException e) { 
+            } catch (IOException e) {
               LOG.warn(e.getMessage());
             }
           }
@@ -1144,7 +1145,7 @@ public class HRegion implements HeapSize
                 // so we do not lose data
                 throw new DroppedSnapshotException("Failed clearing memory after " +
                   actualFlushes + " attempts on region: " + Bytes.toStringBinary(getRegionName()));
-              } 
+              }
               LOG.info("Running extra flush, " + actualFlushes +
                 " (carrying snapshot?) " + this);
             }
@@ -2786,59 +2787,12 @@ public class HRegion implements HeapSize
    */
   public void addRegionToSnapshot(SnapshotDescription desc,
       ForeignExceptionSnare exnSnare) throws IOException {
-    // This should be "fast" since we don't rewrite store files but instead
-    // back up the store files by creating a reference
-    Path rootDir = FSUtils.getRootDir(this.rsServices.getConfiguration());
+    Path rootDir = FSUtils.getRootDir(conf);
     Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
 
-    // 1. dump region meta info into the snapshot directory
-    LOG.debug("Storing region-info for snapshot.");
-    HRegionFileSystem snapshotRegionFs = HRegionFileSystem.createRegionOnFileSystem(conf,
-        this.fs.getFileSystem(), snapshotDir, getRegionInfo());
-
-    // 2. iterate through all the stores in the region
-    LOG.debug("Creating references for hfiles");
-
-    // This ensures that we have an atomic view of the directory as long as we have < ls limit
-    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
-    // batches and may miss files being added/deleted. This could be more robust (iteratively
-    // checking to see if we have all the files until we are sure), but the limit is currently 1000
-    // files/batch, far more than the number of store files under a single column family.
-    for (Store store : stores.values()) {
-      // 2.1. build the snapshot reference directory for the store
-      Path dstStoreDir = snapshotRegionFs.getStoreDir(store.getFamily().getNameAsString());
-      List<StoreFile> storeFiles = new ArrayList<StoreFile>(store.getStorefiles());
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Adding snapshot references for " + storeFiles  + " hfiles");
-      }
-
-      // 2.2. iterate through all the store's files and create "references".
-      int sz = storeFiles.size();
-      for (int i = 0; i < sz; i++) {
-        if (exnSnare != null) {
-          exnSnare.rethrowException();
-        }
-        StoreFile storeFile = storeFiles.get(i);
-        Path file = storeFile.getPath();
-
-        LOG.debug("Creating reference for file (" + (i+1) + "/" + sz + ") : " + file);
-        Path referenceFile = new Path(dstStoreDir, file.getName());
-        boolean success = true;
-        if (storeFile.isReference()) {
-          // write the Reference object to the snapshot
-          storeFile.getFileInfo().getReference().write(fs.getFileSystem(), referenceFile);
-        } else {
-          // create "reference" to this store file.  It is intentionally an empty file -- all
-          // necessary information is captured by its fs location and filename.  This allows us to
-          // only figure out what needs to be done via a single nn operation (instead of having to
-          // open and read the files as well).
-          success = fs.getFileSystem().createNewFile(referenceFile);
-        }
-        if (!success) {
-          throw new IOException("Failed to create reference file:" + referenceFile);
-        }
-      }
-    }
+    SnapshotManifest manifest = SnapshotManifest.create(conf, getFilesystem(),
+                                                        snapshotDir, desc, exnSnare);
+    manifest.addRegion(this);
   }
 
   /**
@@ -4005,14 +3959,14 @@ public class HRegion implements HeapSize
               isStopRow(nextKv.getRowArray(), nextKv.getRowOffset(), nextKv.getRowLength());
           // save that the row was empty before filters applied to it.
           final boolean isEmptyRow = results.isEmpty();
-          
+
           // We have the part of the row necessary for filtering (all of it, usually).
           // First filter with the filterRow(List).
           FilterWrapper.FilterRowRetCode ret = FilterWrapper.FilterRowRetCode.NOT_CALLED;
           if (filter != null && filter.hasFilterRow()) {
             ret = filter.filterRowCellsWithRet(results);
           }
-          
+
           if ((isEmptyRow || ret == FilterWrapper.FilterRowRetCode.EXCLUDE) || filterRow()) {
             results.clear();
             boolean moreRows = nextRow(currentRow, offset, length);
@@ -4080,7 +4034,7 @@ public class HRegion implements HeapSize
       return filter != null && (!filter.hasFilterRow())
           && filter.filterRow();
     }
-    
+
     private boolean filterRowKey(byte[] row, int offset, short length) throws IOException {
       return filter != null
           && filter.filterRowKey(row, offset, length);
@@ -5743,7 +5697,7 @@ public class HRegion implements HeapSize
    * modifies data. It has to be called just before a try.
    * #closeRegionOperation needs to be called in the try's finally block
    * Acquires a read lock and checks if the region is closing or closed.
-   * @throws IOException 
+   * @throws IOException
    */
   public void startRegionOperation() throws IOException {
     startRegionOperation(Operation.ANY);
@@ -5751,7 +5705,7 @@ public class HRegion implements HeapSize
 
   /**
    * @param op The operation is about to be taken on the region
-   * @throws IOException 
+   * @throws IOException
    */
   protected void startRegionOperation(Operation op) throws IOException {
     switch (op) {
@@ -5801,7 +5755,7 @@ public class HRegion implements HeapSize
   /**
    * Closes the lock. This needs to be called in the finally block corresponding
    * to the try block of #startRegionOperation
-   * @throws IOException 
+   * @throws IOException
    */
   public void closeRegionOperation() throws IOException {
     closeRegionOperation(Operation.ANY);

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java Wed May  7 21:28:12 2014
@@ -77,7 +77,7 @@ public class HRegionFileSystem {
   private final Configuration conf;
   private final Path tableDir;
   private final FileSystem fs;
-  
+
   /**
    * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the
    * client level.
@@ -149,7 +149,7 @@ public class HRegionFileSystem {
    * @param familyName Column Family Name
    * @return {@link Path} to the directory of the specified family
    */
-  Path getStoreDir(final String familyName) {
+  public Path getStoreDir(final String familyName) {
     return new Path(this.getRegionDir(), familyName);
   }
 
@@ -176,20 +176,31 @@ public class HRegionFileSystem {
     return getStoreFiles(Bytes.toString(familyName));
   }
 
+  public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
+    return getStoreFiles(familyName, true);
+  }
+
   /**
    * Returns the store files available for the family.
    * This methods performs the filtering based on the valid store files.
    * @param familyName Column Family Name
    * @return a set of {@link StoreFileInfo} for the specified family.
    */
-  public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
+  public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate)
+      throws IOException {
     Path familyDir = getStoreDir(familyName);
     FileStatus[] files = FSUtils.listStatus(this.fs, familyDir);
-    if (files == null) return null;
+    if (files == null) {
+      LOG.debug("No StoreFiles for: " + familyDir);
+      return null;
+    }
 
     ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
     for (FileStatus status: files) {
-      if (!StoreFileInfo.isValid(status)) continue;
+      if (validate && !StoreFileInfo.isValid(status)) {
+        LOG.warn("Invalid StoreFile: " + status.getPath());
+        continue;
+      }
 
       storeFiles.add(new StoreFileInfo(this.conf, this.fs, status));
     }
@@ -354,7 +365,7 @@ public class HRegionFileSystem {
     Path storeDir = getStoreDir(familyName);
     if(!fs.exists(storeDir) && !createDir(storeDir))
       throw new IOException("Failed creating " + storeDir);
-    
+
     String name = buildPath.getName();
     if (generateNewName) {
       name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
@@ -556,16 +567,16 @@ public class HRegionFileSystem {
    */
   Path splitStoreFile(final HRegionInfo hri, final String familyName,
       final StoreFile f, final byte[] splitRow, final boolean top) throws IOException {
-    
+
     // Check whether the split row lies in the range of the store file
     // If it is outside the range, return directly.
     if (top) {
       //check if larger than last key.
       KeyValue splitKey = KeyValueUtil.createFirstOnRow(splitRow);
-      byte[] lastKey = f.createReader().getLastKey();      
+      byte[] lastKey = f.createReader().getLastKey();
       // If lastKey is null means storefile is empty.
       if (lastKey == null) return null;
-      if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(), 
+      if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
           splitKey.getKeyOffset(), splitKey.getKeyLength(), lastKey, 0, lastKey.length) > 0) {
         return null;
       }
@@ -575,14 +586,14 @@ public class HRegionFileSystem {
       byte[] firstKey = f.createReader().getFirstKey();
       // If firstKey is null means storefile is empty.
       if (firstKey == null) return null;
-      if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(), 
+      if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
           splitKey.getKeyOffset(), splitKey.getKeyLength(), firstKey, 0, firstKey.length) < 0) {
         return null;
-      }      
+      }
     }
- 
+
     f.getReader().close(true);
-    
+
     Path splitDir = new Path(getSplitsDir(hri), familyName);
     // A reference to the bottom half of the hsf store file.
     Reference r =
@@ -681,7 +692,7 @@ public class HRegionFileSystem {
    * Commit a merged region, moving it from the merges temporary directory to
    * the proper location in the filesystem.
    * @param mergedRegionInfo merged region {@link HRegionInfo}
-   * @throws IOException 
+   * @throws IOException
    */
   void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
     Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Wed May  7 21:28:12 2014
@@ -222,7 +222,7 @@ public class StoreFile {
    * @return the StoreFile object associated to this StoreFile.
    *         null if the StoreFile is not a reference.
    */
-  StoreFileInfo getFileInfo() {
+  public StoreFileInfo getFileInfo() {
     return this.fileInfo;
   }
 
@@ -614,7 +614,7 @@ public class StoreFile {
       if (comparator == null) {
         comparator = KeyValue.COMPARATOR;
       }
-      return new Writer(fs, filePath, 
+      return new Writer(fs, filePath,
           conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext);
     }
   }
@@ -694,7 +694,7 @@ public class StoreFile {
 
     /** Bytes per Checksum */
     protected int bytesPerChecksum;
-    
+
     TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
     /* isTimeRangeTrackerSet keeps track if the timeRange has already been set
      * When flushing a memstore, we set TimeRange and use this variable to
@@ -723,7 +723,7 @@ public class StoreFile {
         final Configuration conf,
         CacheConfig cacheConf,
         final KVComparator comparator, BloomType bloomType, long maxKeys,
-        InetSocketAddress[] favoredNodes, HFileContext fileContext) 
+        InetSocketAddress[] favoredNodes, HFileContext fileContext)
             throws IOException {
       writer = HFile.getWriterFactory(conf, cacheConf)
           .withPath(fs, path)

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java?rev=1593139&r1=1593138&r2=1593139&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java Wed May  7 21:28:12 2014
@@ -144,7 +144,7 @@ public class StoreFileInfo {
    * @return the Reference object associated to this StoreFileInfo.
    *         null if the StoreFile is not a reference.
    */
-  Reference getReference() {
+  public Reference getReference() {
     return this.reference;
   }
 



Mime
View raw message