Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ADC0EFEC6 for ; Fri, 5 Apr 2013 11:42:55 +0000 (UTC) Received: (qmail 62896 invoked by uid 500); 5 Apr 2013 11:42:55 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 62823 invoked by uid 500); 5 Apr 2013 11:42:55 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 62815 invoked by uid 99); 5 Apr 2013 11:42:55 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Apr 2013 11:42:55 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Apr 2013 11:42:52 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8EE3523888E4; Fri, 5 Apr 2013 11:42:32 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1464931 - in /hbase/branches/0.94/src: main/jamon/org/apache/hadoop/hbase/tmpl/master/ main/java/org/apache/hadoop/hbase/master/ main/java/org/apache/hadoop/hbase/master/metrics/ main/java/org/apache/hadoop/hbase/master/snapshot/ main/java... Date: Fri, 05 Apr 2013 11:42:31 -0000 To: commits@hbase.apache.org From: mbertozzi@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130405114232.8EE3523888E4@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mbertozzi Date: Fri Apr 5 11:42:31 2013 New Revision: 1464931 URL: http://svn.apache.org/r1464931 Log: HBASE-7615 Add metrics for snapshots Added: hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp Modified: hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java Modified: hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon (original) +++ hbase/branches/0.94/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon Fri Apr 5 11:42:31 2013 @@ -212,7 +212,7 @@ org.apache.hadoop.hbase.protobuf.generat <%for SnapshotDescription snapshotDesc : snapshots%> - <% snapshotDesc.getName() %> + <% snapshotDesc.getName() %> <% snapshotDesc.getTable() %> <% new Date(snapshotDesc.getCreationTime()) %> <% snapshotDesc.getType() %> Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Fri Apr 5 11:42:31 2013 @@ -492,7 +492,7 @@ Server { ", cluster-up flag was=" + wasUp); // create the snapshot manager - this.snapshotManager = new SnapshotManager(this); + this.snapshotManager = new SnapshotManager(this, this.metrics); } // Check if we should stop every second. Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetrics.java Fri Apr 5 11:42:31 2013 @@ -50,7 +50,8 @@ public class MasterMetrics implements Up private long lastUpdate = System.currentTimeMillis(); private long lastExtUpdate = System.currentTimeMillis(); private long extendedPeriod = 0; -/* + + /* * Count of requests to the cluster since last call to metrics update */ private final MetricsRate cluster_requests = @@ -64,6 +65,18 @@ public class MasterMetrics implements Up final PersistentMetricsTimeVaryingRate splitSize = new PersistentMetricsTimeVaryingRate("splitSize", registry); + /** Time it takes to finish snapshot() */ + final PersistentMetricsTimeVaryingRate snapshotTime = + new PersistentMetricsTimeVaryingRate("snapshotTime", registry); + + /** Time it takes to finish restoreSnapshot() */ + final PersistentMetricsTimeVaryingRate snapshotRestoreTime = + new PersistentMetricsTimeVaryingRate("snapshotRestoreTime", registry); + + /** Time it takes to finish cloneSnapshotTime() */ + final PersistentMetricsTimeVaryingRate snapshotCloneTime = + new PersistentMetricsTimeVaryingRate("snapshotCloneTime", registry); + public MasterMetrics(final String name) { MetricsContext context = MetricsUtil.getContext("hbase"); metricsRecord = MetricsUtil.createRecord(context, "master"); @@ -146,4 +159,28 @@ public class MasterMetrics implements Up public void incrementRequests(final int inc) { this.cluster_requests.inc(inc); } + + /** + * Record a single instance of a snapshot + * @param time time that the snapshot took + */ + public void addSnapshot(long time) { + snapshotTime.inc(time); + } + + /** + * Record a single instance of a snapshot + * @param time time that the snapshot restore took + */ + public void addSnapshotRestore(long time) { + snapshotRestoreTime.inc(time); + } + + /** + * Record a single instance of a snapshot cloned table + * @param time time that the snapshot clone took + */ + public void addSnapshotClone(long time) { + snapshotCloneTime.inc(time); + } } Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/CloneSnapshotHandler.java Fri Apr 5 11:42:31 2013 @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.errorhand import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.SnapshotSentinel; import org.apache.hadoop.hbase.master.handler.CreateTableHandler; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; @@ -63,17 +64,20 @@ public class CloneSnapshotHandler extend private final SnapshotDescription snapshot; private final ForeignExceptionDispatcher monitor; + private final MasterMetrics metricsMaster; private final MonitoredTask status; private volatile boolean stopped = false; public CloneSnapshotHandler(final MasterServices masterServices, - final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor) + final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor, + final MasterMetrics metricsMaster) throws NotAllMetaRegionsOnlineException, TableExistsException, IOException { - super(masterServices, masterServices.getMasterFileSystem(), + super(masterServices, masterServices.getMasterFileSystem(), masterServices.getServerManager(), hTableDescriptor, masterServices.getConfiguration(), null, masterServices.getCatalogTracker(), masterServices.getAssignmentManager()); + this.metricsMaster = metricsMaster; // Snapshot information this.snapshot = snapshot; @@ -137,6 +141,7 @@ public class CloneSnapshotHandler extend } else { status.markComplete("Snapshot '"+ snapshot.getName() +"' clone completed and table enabled!"); } + metricsMaster.addSnapshotClone(status.getCompletionTimestamp() - status.getStartTime()); super.completed(exception); } Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java Fri Apr 5 11:42:31 2013 @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.ServerNam import org.apache.hadoop.hbase.errorhandling.ForeignException; import org.apache.hadoop.hbase.errorhandling.TimeoutExceptionInjector; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; @@ -62,8 +63,8 @@ public class DisabledTableSnapshotHandle * @throws IOException on unexpected error */ public DisabledTableSnapshotHandler(SnapshotDescription snapshot, - final MasterServices masterServices) throws IOException { - super(snapshot, masterServices); + final MasterServices masterServices, final MasterMetrics metricsMaster) throws IOException { + super(snapshot, masterServices, metricsMaster); // setup the timer timeoutInjector = TakeSnapshotUtils.getMasterTimerAndBindToMonitor(snapshot, conf, monitor); Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java Fri Apr 5 11:42:31 2013 @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HRegionIn import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.errorhandling.ForeignException; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; import org.apache.hadoop.hbase.procedure.Procedure; import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; @@ -49,8 +50,8 @@ public class EnabledTableSnapshotHandler private final ProcedureCoordinator coordinator; public EnabledTableSnapshotHandler(SnapshotDescription snapshot, MasterServices master, - SnapshotManager manager) throws IOException { - super(snapshot, master); + final SnapshotManager manager, final MasterMetrics metricsMaster) throws IOException { + super(snapshot, master, metricsMaster); this.coordinator = manager.getCoordinator(); } Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/RestoreSnapshotHandler.java Fri Apr 5 11:42:31 2013 @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.master.Ma import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.SnapshotSentinel; import org.apache.hadoop.hbase.master.handler.TableEventHandler; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; @@ -60,14 +61,16 @@ public class RestoreSnapshotHandler exte private final SnapshotDescription snapshot; private final ForeignExceptionDispatcher monitor; + private final MasterMetrics metricsMaster; private final MonitoredTask status; private volatile boolean stopped = false; public RestoreSnapshotHandler(final MasterServices masterServices, - final SnapshotDescription snapshot, final HTableDescriptor htd) - throws IOException { + final SnapshotDescription snapshot, final HTableDescriptor htd, + final MasterMetrics metricsMaster) throws IOException { super(EventType.C_M_RESTORE_SNAPSHOT, htd.getName(), masterServices, masterServices); + this.metricsMaster = metricsMaster; // Snapshot information this.snapshot = snapshot; @@ -146,6 +149,8 @@ public class RestoreSnapshotHandler exte } else { status.markComplete("Restore snapshot '"+ snapshot.getName() +"' completed!"); } + metricsMaster.addSnapshotRestore(status.getCompletionTimestamp() - status.getStartTime()); + super.completed(exception); } @Override Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java Fri Apr 5 11:42:31 2013 @@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.master.Ma import org.apache.hadoop.hbase.master.SnapshotSentinel; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; import org.apache.hadoop.hbase.procedure.Procedure; import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs; @@ -117,6 +118,7 @@ public class SnapshotManager implements private boolean stopped; private final long wakeFrequency; private final MasterServices master; // Needed by TableEventHandlers + private final MasterMetrics metricsMaster; private final ProcedureCoordinator coordinator; // Is snapshot feature enabled? @@ -138,9 +140,11 @@ public class SnapshotManager implements * Construct a snapshot manager. * @param master */ - public SnapshotManager(final MasterServices master) throws KeeperException, IOException, - UnsupportedOperationException { + public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster) + throws KeeperException, IOException, UnsupportedOperationException { this.master = master; + this.metricsMaster = metricsMaster; + checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem()); // get the configuration for the coordinator @@ -165,9 +169,12 @@ public class SnapshotManager implements * @param coordinator procedure coordinator instance. exposed for testing. * @param pool HBase ExecutorServcie instance, exposed for testing. */ - public SnapshotManager(final MasterServices master, ProcedureCoordinator coordinator, ExecutorService pool) + public SnapshotManager(final MasterServices master, final MasterMetrics metricsMaster, + ProcedureCoordinator coordinator, ExecutorService pool) throws IOException, UnsupportedOperationException { this.master = master; + this.metricsMaster = metricsMaster; + checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem()); this.wakeFrequency = master.getConfiguration().getInt(SNAPSHOT_WAKE_MILLIS_KEY, @@ -427,7 +434,7 @@ public class SnapshotManager implements throws HBaseSnapshotException { TakeSnapshotHandler handler; try { - handler = new EnabledTableSnapshotHandler(snapshot, master, this); + handler = new EnabledTableSnapshotHandler(snapshot, master, this, metricsMaster); this.executorService.submit(handler); this.handler = handler; } catch (IOException e) { @@ -536,7 +543,7 @@ public class SnapshotManager implements DisabledTableSnapshotHandler handler; try { - handler = new DisabledTableSnapshotHandler(snapshot, this.master); + handler = new DisabledTableSnapshotHandler(snapshot, master, metricsMaster); this.executorService.submit(handler); this.handler = handler; } catch (IOException e) { @@ -619,7 +626,7 @@ public class SnapshotManager implements try { CloneSnapshotHandler handler = - new CloneSnapshotHandler(master, snapshot, hTableDescriptor); + new CloneSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster); this.executorService.submit(handler); restoreHandlers.put(tableName, handler); } catch (Exception e) { @@ -710,7 +717,7 @@ public class SnapshotManager implements try { RestoreSnapshotHandler handler = - new RestoreSnapshotHandler(master, snapshot, hTableDescriptor); + new RestoreSnapshotHandler(master, snapshot, hTableDescriptor, metricsMaster); this.executorService.submit(handler); restoreHandlers.put(hTableDescriptor.getNameAsString(), handler); } catch (Exception e) { Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java Fri Apr 5 11:42:31 2013 @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.errorhand import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.SnapshotSentinel; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; @@ -66,6 +67,7 @@ public abstract class TakeSnapshotHandle // none of these should ever be null protected final MasterServices master; + protected final MasterMetrics metricsMaster; protected final SnapshotDescription snapshot; protected final Configuration conf; protected final FileSystem fs; @@ -81,13 +83,14 @@ public abstract class TakeSnapshotHandle * @param masterServices master services provider * @throws IOException on unexpected error */ - public TakeSnapshotHandler(SnapshotDescription snapshot, - final MasterServices masterServices) throws IOException { + public TakeSnapshotHandler(SnapshotDescription snapshot, final MasterServices masterServices, + final MasterMetrics metricsMaster) throws IOException { super(masterServices, EventType.C_M_SNAPSHOT_TABLE); assert snapshot != null : "SnapshotDescription must not be nul1"; assert masterServices != null : "MasterServices must not be nul1"; this.master = masterServices; + this.metricsMaster = metricsMaster; this.snapshot = snapshot; this.conf = this.master.getConfiguration(); this.fs = this.master.getMasterFileSystem().getFileSystem(); @@ -156,6 +159,7 @@ public abstract class TakeSnapshotHandle completeSnapshot(this.snapshotDir, this.workingDir, this.fs); status.markComplete("Snapshot " + snapshot.getName() + " of table " + snapshot.getTable() + " completed"); + metricsMaster.addSnapshot(status.getCompletionTimestamp() - status.getStartTime()); } catch (Exception e) { status.abort("Failed to complete snapshot " + snapshot.getName() + " on table " + snapshot.getTable() + " because " + e.getMessage()); Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java (original) +++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java Fri Apr 5 11:42:31 2013 @@ -63,6 +63,187 @@ import org.apache.hadoop.hbase.util.FSTa public final class SnapshotInfo extends Configured implements Tool { private static final Log LOG = LogFactory.getLog(SnapshotInfo.class); + /** + * Statistics about the snapshot + *
    + *
  1. How many store files and logs are in the archive + *
  2. How many store files and logs are shared with the table + *
  3. Total store files and logs size and shared amount + *
+ */ + public static class SnapshotStats { + /** Information about the file referenced by the snapshot */ + static class FileInfo { + private final boolean inArchive; + private final long size; + + FileInfo(final boolean inArchive, final long size) { + this.inArchive = inArchive; + this.size = size; + } + + /** @return true if the file is in the archive */ + public boolean inArchive() { + return this.inArchive; + } + + /** @return true if the file is missing */ + public boolean isMissing() { + return this.size < 0; + } + + /** @return the file size */ + public long getSize() { + return this.size; + } + } + + private int hfileArchiveCount = 0; + private int hfilesMissing = 0; + private int hfilesCount = 0; + private int logsMissing = 0; + private int logsCount = 0; + private long hfileArchiveSize = 0; + private long hfileSize = 0; + private long logSize = 0; + + private final SnapshotDescription snapshot; + private final Configuration conf; + private final FileSystem fs; + + SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot) + { + this.snapshot = snapshot; + this.conf = conf; + this.fs = fs; + } + + /** @return the snapshot descriptor */ + public SnapshotDescription getSnapshotDescription() { + return this.snapshot; + } + + /** @return true if the snapshot is corrupted */ + public boolean isSnapshotCorrupted() { + return hfilesMissing > 0 || logsMissing > 0; + } + + /** @return the number of available store files */ + public int getStoreFilesCount() { + return hfilesCount + hfileArchiveCount; + } + + /** @return the number of available store files in the archive */ + public int getArchivedStoreFilesCount() { + return hfileArchiveCount; + } + + /** @return the number of available log files */ + public int getLogsCount() { + return logsCount; + } + + /** @return the number of missing store files */ + public int getMissingStoreFilesCount() { + return hfilesMissing; + } + + /** @return the number of missing log files */ + public int getMissingLogsCount() { + return logsMissing; + } + + /** @return the total size of the store files referenced by the snapshot */ + public long getStoreFilesSize() { + return hfileSize + hfileArchiveSize; + } + + /** @return the total size of the store files shared */ + public long getSharedStoreFilesSize() { + return hfileSize; + } + + /** @return the total size of the store files in the archive */ + public long getArchivedStoreFileSize() { + return hfileArchiveSize; + } + + /** @return the percentage of the shared store files */ + public float getSharedStoreFilePercentage() { + return ((float)hfileSize / (hfileSize + hfileArchiveSize)) * 100; + } + + /** @return the total log size */ + public long getLogsSize() { + return logSize; + } + + /** + * Add the specified store file to the stats + * @param region region encoded Name + * @param family family name + * @param hfile store file name + * @return the store file information + */ + FileInfo addStoreFile(final String region, final String family, final String hfile) + throws IOException { + String table = this.snapshot.getTable(); + Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); + HFileLink link = new HFileLink(conf, path); + boolean inArchive = false; + long size = -1; + try { + if ((inArchive = fs.exists(link.getArchivePath()))) { + size = fs.getFileStatus(link.getArchivePath()).getLen(); + hfileArchiveSize += size; + hfileArchiveCount++; + } else { + size = link.getFileStatus(fs).getLen(); + hfileSize += size; + hfilesCount++; + } + } catch (FileNotFoundException e) { + hfilesMissing++; + } + return new FileInfo(inArchive, size); + } + + /** + * Add the specified recovered.edits file to the stats + * @param region region encoded name + * @param logfile log file name + * @return the recovered.edits information + */ + FileInfo addRecoveredEdits(final String region, final String logfile) throws IOException { + Path rootDir = FSUtils.getRootDir(conf); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); + Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile); + long size = fs.getFileStatus(path).getLen(); + logSize += size; + logsCount++; + return new FileInfo(true, size); + } + + /** + * Add the specified log file to the stats + * @param server server name + * @param logfile log file name + * @return the log information + */ + FileInfo addLogFile(final String server, final String logfile) throws IOException { + HLogLink logLink = new HLogLink(conf, server, logfile); + long size = -1; + try { + size = logLink.getFileStatus(fs).getLen(); + logSize += size; + logsCount++; + } catch (FileNotFoundException e) { + logsMissing++; + } + return new FileInfo(false, size); + } + } + private FileSystem fs; private Path rootDir; @@ -170,104 +351,68 @@ public final class SnapshotInfo extends * dump the file list if requested and the collected information. */ private void printFiles(final boolean showFiles) throws IOException { - final String table = snapshotDesc.getTable(); - final Configuration conf = getConf(); - if (showFiles) { System.out.println("Snapshot Files"); System.out.println("----------------------------------------"); } // Collect information about hfiles and logs in the snapshot - final AtomicInteger hfileArchiveCount = new AtomicInteger(); - final AtomicInteger hfilesMissing = new AtomicInteger(); - final AtomicInteger hfilesCount = new AtomicInteger(); - final AtomicInteger logsMissing = new AtomicInteger(); - final AtomicInteger logsCount = new AtomicInteger(); - final AtomicLong hfileArchiveSize = new AtomicLong(); - final AtomicLong hfileSize = new AtomicLong(); - final AtomicLong logSize = new AtomicLong(); + final String table = this.snapshotDesc.getTable(); + final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, this.snapshotDesc); SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir, new SnapshotReferenceUtil.FileVisitor() { public void storeFile (final String region, final String family, final String hfile) throws IOException { - Path path = new Path(family, HFileLink.createHFileLinkName(table, region, hfile)); - HFileLink link = new HFileLink(conf, path); - boolean inArchive = false; - long size = -1; - try { - if ((inArchive = fs.exists(link.getArchivePath()))) { - size = fs.getFileStatus(link.getArchivePath()).getLen(); - hfileArchiveSize.addAndGet(size); - hfileArchiveCount.addAndGet(1); - } else { - size = link.getFileStatus(fs).getLen(); - hfileSize.addAndGet(size); - hfilesCount.addAndGet(1); - } - } catch (FileNotFoundException e) { - hfilesMissing.addAndGet(1); - } + SnapshotStats.FileInfo info = stats.addStoreFile(region, family, hfile); if (showFiles) { System.out.printf("%8s %s/%s/%s/%s %s%n", - (size < 0 ? "-" : StringUtils.humanReadableInt(size)), + (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())), table, region, family, hfile, - (inArchive ? "(archive)" : (size < 0) ? "(NOT FOUND)" : "")); + (info.inArchive() ? "(archive)" : info.isMissing() ? "(NOT FOUND)" : "")); } } public void recoveredEdits (final String region, final String logfile) throws IOException { - Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile); - long size = fs.getFileStatus(path).getLen(); - logSize.addAndGet(size); - logsCount.addAndGet(1); + SnapshotStats.FileInfo info = stats.addRecoveredEdits(region, logfile); if (showFiles) { System.out.printf("%8s recovered.edits %s on region %s%n", - StringUtils.humanReadableInt(size), logfile, region); + StringUtils.humanReadableInt(info.getSize()), logfile, region); } } public void logFile (final String server, final String logfile) throws IOException { - HLogLink logLink = new HLogLink(conf, server, logfile); - long size = -1; - try { - size = logLink.getFileStatus(fs).getLen(); - logSize.addAndGet(size); - logsCount.addAndGet(1); - } catch (FileNotFoundException e) { - logsMissing.addAndGet(1); - } + SnapshotStats.FileInfo info = stats.addLogFile(server, logfile); if (showFiles) { System.out.printf("%8s log %s on server %s %s%n", - (size < 0 ? "-" : StringUtils.humanReadableInt(size)), + (info.isMissing() ? "-" : StringUtils.humanReadableInt(info.getSize())), logfile, server, - (size < 0 ? "(NOT FOUND)" : "")); + (info.isMissing() ? "(NOT FOUND)" : "")); } } }); // Dump the stats System.out.println(); - if (hfilesMissing.get() > 0 || logsMissing.get() > 0) { + if (stats.isSnapshotCorrupted()) { System.out.println("**************************************************************"); System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n", - hfilesMissing.get(), logsMissing.get()); + stats.getMissingStoreFilesCount(), stats.getMissingLogsCount()); System.out.println("**************************************************************"); } System.out.printf("%d HFiles (%d in archive), total size %s (%.2f%% %s shared with the source table)%n", - hfilesCount.get() + hfileArchiveCount.get(), hfileArchiveCount.get(), - StringUtils.humanReadableInt(hfileSize.get() + hfileArchiveSize.get()), - ((float)hfileSize.get() / (hfileSize.get() + hfileArchiveSize.get())) * 100, - StringUtils.humanReadableInt(hfileSize.get()) + stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), + StringUtils.humanReadableInt(stats.getStoreFilesSize()), + stats.getSharedStoreFilePercentage(), + StringUtils.humanReadableInt(stats.getSharedStoreFilesSize()) ); System.out.printf("%d Logs, total size %s%n", - logsCount.get(), StringUtils.humanReadableInt(logSize.get())); + stats.getLogsCount(), StringUtils.humanReadableInt(stats.getLogsSize())); System.out.println(); } @@ -287,6 +432,36 @@ public final class SnapshotInfo extends } /** + * Returns the snapshot stats + * @param conf the {@link Configuration} to use + * @param snapshot {@link SnapshotDescription} to get stats from + * @return the snapshot stats + */ + public static SnapshotStats getSnapshotStats(final Configuration conf, + final SnapshotDescription snapshot) throws IOException { + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = FileSystem.get(conf); + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); + final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot); + SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir, + new SnapshotReferenceUtil.FileVisitor() { + public void storeFile (final String region, final String family, final String hfile) + throws IOException { + stats.addStoreFile(region, family, hfile); + } + + public void recoveredEdits (final String region, final String logfile) throws IOException { + stats.addRecoveredEdits(region, logfile); + } + + public void logFile (final String server, final String logfile) throws IOException { + stats.addLogFile(server, logfile); + } + }); + return stats; + } + + /** * The guts of the {@link #main} method. * Call this method to avoid the {@link #main(String[])} System.exit. * @param args Added: hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp?rev=1464931&view=auto ============================================================================== --- hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp (added) +++ hbase/branches/0.94/src/main/resources/hbase-webapps/master/snapshot.jsp Fri Apr 5 11:42:31 2013 @@ -0,0 +1,189 @@ +<%-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--%> +<%@ page contentType="text/html;charset=UTF-8" + import="java.util.Date" + import="java.util.HashMap" + import="org.apache.hadoop.conf.Configuration" + import="org.apache.hadoop.hbase.client.HBaseAdmin" + import="org.apache.hadoop.hbase.client.HConnectionManager" + import="org.apache.hadoop.hbase.HRegionInfo" + import="org.apache.hadoop.hbase.master.HMaster" + import="org.apache.hadoop.hbase.util.Bytes" + import="org.apache.hadoop.hbase.util.FSUtils" + import="org.apache.hadoop.hbase.protobuf.ProtobufUtil" + import="org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription" + import="org.apache.hadoop.hbase.snapshot.SnapshotInfo" + import="org.apache.hadoop.util.StringUtils" + import="java.util.List" + import="java.util.Map" + import="org.apache.hadoop.hbase.HConstants"%><% + HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); + Configuration conf = master.getConfiguration(); + HBaseAdmin hbadmin = new HBaseAdmin(conf); + boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false); + String snapshotName = request.getParameter("name"); + SnapshotDescription snapshot = null; + SnapshotInfo.SnapshotStats stats = null; + for (SnapshotDescription snapshotDesc: hbadmin.listSnapshots()) { + if (snapshotName.equals(snapshotDesc.getName())) { + snapshot = snapshotDesc; + stats = SnapshotInfo.getSnapshotStats(conf, snapshot); + break; + } + } + + String action = request.getParameter("action"); + String cloneName = request.getParameter("cloneName"); + boolean isActionResultPage = (!readOnly && action != null); +%> + + + + + + +<% if (isActionResultPage) { %> + HBase Master: <%= master.getServerName() %> + +<% } else { %> + Snapshot: <%= snapshotName %> +<% } %> + + + +<% if (isActionResultPage) { %> +

Snapshot action request...

+<% + if (action.equals("restore")) { + hbadmin.restoreSnapshot(snapshotName); + %> Restore Snapshot request accepted. <% + } else if (action.equals("clone")) { + if (cloneName != null && cloneName.length() > 0) { + hbadmin.cloneSnapshot(snapshotName, cloneName); + %> Clone from Snapshot request accepted. <% + } else { + %> Clone from Snapshot request failed, No table name specified. <% + } + } +%> +

Go Back, or wait for the redirect. + +<% } else if (snapshot == null) { %> +

Snapshot "<%= snapshotName %>" does not exists

+ +
+

Go Back, or wait for the redirect. +<% } else { %> +

Snapshot: <%= snapshotName %>

+ +
+

Snapshot Attributes

+ + + + + + + + + + + + + + <% if (stats.isSnapshotCorrupted()) { %> + + <% } else { %> + + <% } %> + +
TableCreation TimeTypeFormat VersionState
<%= snapshot.getTable() %><%= new Date(snapshot.getCreationTime()) %><%= snapshot.getType() %><%= snapshot.getVersion() %>CORRUPTEDok
+

+ <%= stats.getStoreFilesCount() %> HFiles (<%= stats.getArchivedStoreFilesCount() %> in archive), + total size <%= StringUtils.humanReadableInt(stats.getStoreFilesSize()) %> + (<%= stats.getSharedStoreFilePercentage() %>% + <%= StringUtils.humanReadableInt(stats.getSharedStoreFilesSize()) %> shared with the source + table) +

+

+ <%= stats.getLogsCount() %> Logs, total size + <%= StringUtils.humanReadableInt(stats.getLogsSize()) %> +

+ <% if (stats.isSnapshotCorrupted()) { %> +

CORRUPTED Snapshot

+

+ <%= stats.getMissingStoreFilesCount() %> hfile(s) and + <%= stats.getMissingLogsCount() %> log(s) missing. +

+ <% } %> +<% + } // end else + +HConnectionManager.deleteConnection(hbadmin.getConfiguration()); +%> + + +<% if (!readOnly && action == null && snapshot != null) { %> +

+Actions: +

+

+ + + + + + + + + + + + + + + + + + + + + + +
+  New Table Name (clone): + This action will create a new table by cloning the snapshot content. + There are no copies of data involved. + And writing on the newly created table will not influence the snapshot data. +
 
+   Restore a specified snapshot. + The restore will replace the content of the original table, + bringing back the content to the snapshot state. + The table must be disabled.
+
+

+ +<% } %> + + Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java?rev=1464931&r1=1464930&r2=1464931&view=diff ============================================================================== --- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java (original) +++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/snapshot/TestSnapshotManager.java Fri Apr 5 11:42:31 2013 @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.master.Ma import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.zookeeper.KeeperException; @@ -49,6 +50,7 @@ public class TestSnapshotManager { private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); MasterServices services = Mockito.mock(MasterServices.class); + MasterMetrics metrics = Mockito.mock(MasterMetrics.class); ProcedureCoordinator coordinator = Mockito.mock(ProcedureCoordinator.class); ExecutorService pool = Mockito.mock(ExecutorService.class); MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class); @@ -71,7 +73,7 @@ public class TestSnapshotManager { Mockito.when(services.getMasterFileSystem()).thenReturn(mfs); Mockito.when(mfs.getFileSystem()).thenReturn(fs); Mockito.when(mfs.getRootDir()).thenReturn(UTIL.getDataTestDir()); - return new SnapshotManager(services, coordinator, pool); + return new SnapshotManager(services, metrics, coordinator, pool); } @Test