Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id BAA7C200CFC for ; Thu, 28 Sep 2017 17:14:34 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B91D91609ED; Thu, 28 Sep 2017 15:14:34 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2FB9B1609C2 for ; Thu, 28 Sep 2017 17:14:33 +0200 (CEST) Received: (qmail 77956 invoked by uid 500); 28 Sep 2017 15:14:13 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 76488 invoked by uid 99); 28 Sep 2017 15:14:12 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Sep 2017 15:14:12 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C46C6F5BD3; Thu, 28 Sep 2017 15:14:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Thu, 28 Sep 2017 15:14:40 -0000 Message-Id: In-Reply-To: <0ae50d3876544e89813c25027ef0caaa@git.apache.org> References: <0ae50d3876544e89813c25027ef0caaa@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [32/51] [partial] hbase-site git commit: Published site at . archived-at: Thu, 28 Sep 2017 15:14:34 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/67deb422/apidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotInfo.html ---------------------------------------------------------------------- diff --git a/apidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotInfo.html b/apidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotInfo.html index 251d4fe..6c97088 100644 --- a/apidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotInfo.html +++ b/apidocs/src-html/org/apache/hadoop/hbase/snapshot/SnapshotInfo.html @@ -26,8 +26,8 @@ 018 019package org.apache.hadoop.hbase.snapshot; 020 -021import java.io.IOException; -022import java.io.FileNotFoundException; +021import java.io.FileNotFoundException; +022import java.io.IOException; 023import java.net.URI; 024import java.text.SimpleDateFormat; 025import java.util.ArrayList; @@ -44,679 +44,677 @@ 036import org.apache.commons.cli.Option; 037import org.apache.commons.logging.Log; 038import org.apache.commons.logging.LogFactory; -039 -040import org.apache.hadoop.fs.Path; -041import org.apache.hadoop.fs.FileStatus; -042import org.apache.hadoop.fs.FileSystem; -043import org.apache.yetus.audience.InterfaceAudience; -044import org.apache.hadoop.hbase.client.SnapshotDescription; -045import org.apache.hadoop.hbase.HRegionInfo; -046import org.apache.hadoop.hbase.TableName; -047import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; +039import org.apache.hadoop.conf.Configuration; +040import org.apache.hadoop.fs.FileStatus; +041import org.apache.hadoop.fs.FileSystem; +042import org.apache.hadoop.fs.Path; +043import org.apache.hadoop.hbase.TableName; +044import org.apache.hadoop.hbase.client.RegionInfo; +045import org.apache.hadoop.hbase.client.SnapshotDescription; +046import org.apache.hadoop.hbase.io.HFileLink; +047import org.apache.hadoop.hbase.io.WALLink; 048import org.apache.hadoop.hbase.util.AbstractHBaseTool; -049import org.apache.hadoop.util.StringUtils; -050 -051import org.apache.hadoop.conf.Configuration; -052import org.apache.hadoop.hbase.io.HFileLink; -053import org.apache.hadoop.hbase.io.WALLink; -054import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -055import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -056import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -057import org.apache.hadoop.hbase.util.FSUtils; -058 -059/** -060 * Tool for dumping snapshot information. -061 * <ol> -062 * <li> Table Descriptor -063 * <li> Snapshot creation time, type, format version, ... -064 * <li> List of hfiles and wals -065 * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ... -066 * </ol> -067 */ -068@InterfaceAudience.Public -069public final class SnapshotInfo extends AbstractHBaseTool { -070 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class); -071 -072 static final class Options { -073 static final Option SNAPSHOT = new Option(null, "snapshot", true, "Snapshot to examine."); -074 static final Option REMOTE_DIR = new Option(null, "remote-dir", true, -075 "Root directory that contains the snapshots."); -076 static final Option LIST_SNAPSHOTS = new Option(null, "list-snapshots", false, -077 "List all the available snapshots and exit."); -078 static final Option FILES = new Option(null, "files", false, "Files and logs list."); -079 static final Option STATS = new Option(null, "stats", false, "Files and logs stats."); -080 static final Option SCHEMA = new Option(null, "schema", false, -081 "Describe the snapshotted table."); -082 static final Option SIZE_IN_BYTES = new Option(null, "size-in-bytes", false, -083 "Print the size of the files in bytes."); -084 } -085 -086 /** -087 * Statistics about the snapshot -088 * <ol> -089 * <li> How many store files and logs are in the archive -090 * <li> How many store files and logs are shared with the table -091 * <li> Total store files and logs size and shared amount -092 * </ol> -093 */ -094 public static class SnapshotStats { -095 /** Information about the file referenced by the snapshot */ -096 static class FileInfo { -097 private final boolean corrupted; -098 private final boolean inArchive; -099 private final long size; -100 -101 FileInfo(final boolean inArchive, final long size, final boolean corrupted) { -102 this.corrupted = corrupted; -103 this.inArchive = inArchive; -104 this.size = size; -105 } -106 -107 /** @return true if the file is in the archive */ -108 public boolean inArchive() { -109 return this.inArchive; -110 } -111 -112 /** @return true if the file is corrupted */ -113 public boolean isCorrupted() { -114 return this.corrupted; -115 } -116 -117 /** @return true if the file is missing */ -118 public boolean isMissing() { -119 return this.size < 0; -120 } -121 -122 /** @return the file size */ -123 public long getSize() { -124 return this.size; -125 } -126 -127 String getStateToString() { -128 if (isCorrupted()) return "CORRUPTED"; -129 if (isMissing()) return "NOT FOUND"; -130 if (inArchive()) return "archive"; -131 return null; -132 } -133 } -134 -135 private AtomicInteger hfilesArchiveCount = new AtomicInteger(); -136 private AtomicInteger hfilesCorrupted = new AtomicInteger(); -137 private AtomicInteger hfilesMissing = new AtomicInteger(); -138 private AtomicInteger hfilesCount = new AtomicInteger(); -139 private AtomicInteger hfilesMobCount = new AtomicInteger(); -140 private AtomicInteger logsMissing = new AtomicInteger(); -141 private AtomicInteger logsCount = new AtomicInteger(); -142 private AtomicLong hfilesArchiveSize = new AtomicLong(); -143 private AtomicLong hfilesSize = new AtomicLong(); -144 private AtomicLong hfilesMobSize = new AtomicLong(); -145 private AtomicLong nonSharedHfilesArchiveSize = new AtomicLong(); -146 private AtomicLong logSize = new AtomicLong(); -147 -148 private final SnapshotProtos.SnapshotDescription snapshot; -149 private final TableName snapshotTable; -150 private final Configuration conf; -151 private final FileSystem fs; -152 -153 SnapshotStats(final Configuration conf, final FileSystem fs, -154 final SnapshotDescription snapshot) -155 { -156 this.snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); -157 this.snapshotTable = snapshot.getTableName(); -158 this.conf = conf; -159 this.fs = fs; -160 } -161 -162 SnapshotStats(final Configuration conf, final FileSystem fs, -163 final SnapshotProtos.SnapshotDescription snapshot) { -164 this.snapshot = snapshot; -165 this.snapshotTable = TableName.valueOf(snapshot.getTable()); -166 this.conf = conf; -167 this.fs = fs; -168 } -169 -170 -171 /** @return the snapshot descriptor */ -172 public SnapshotDescription getSnapshotDescription() { -173 return ProtobufUtil.createSnapshotDesc(this.snapshot); -174 } -175 -176 /** @return true if the snapshot is corrupted */ -177 public boolean isSnapshotCorrupted() { -178 return hfilesMissing.get() > 0 || -179 logsMissing.get() > 0 || -180 hfilesCorrupted.get() > 0; -181 } -182 -183 /** @return the number of available store files */ -184 public int getStoreFilesCount() { -185 return hfilesCount.get() + hfilesArchiveCount.get() + hfilesMobCount.get(); -186 } -187 -188 /** @return the number of available store files in the archive */ -189 public int getArchivedStoreFilesCount() { -190 return hfilesArchiveCount.get(); -191 } -192 -193 /** @return the number of available store files in the mob dir */ -194 public int getMobStoreFilesCount() { return hfilesMobCount.get(); } -195 -196 /** @return the number of available log files */ -197 public int getLogsCount() { -198 return logsCount.get(); -199 } -200 -201 /** @return the number of missing store files */ -202 public int getMissingStoreFilesCount() { -203 return hfilesMissing.get(); -204 } -205 -206 /** @return the number of corrupted store files */ -207 public int getCorruptedStoreFilesCount() { -208 return hfilesCorrupted.get(); -209 } -210 -211 /** @return the number of missing log files */ -212 public int getMissingLogsCount() { -213 return logsMissing.get(); -214 } -215 -216 /** @return the total size of the store files referenced by the snapshot */ -217 public long getStoreFilesSize() { -218 return hfilesSize.get() + hfilesArchiveSize.get() + hfilesMobSize.get(); -219 } -220 -221 /** @return the total size of the store files shared */ -222 public long getSharedStoreFilesSize() { -223 return hfilesSize.get(); -224 } -225 -226 /** @return the total size of the store files in the archive */ -227 public long getArchivedStoreFileSize() { -228 return hfilesArchiveSize.get(); -229 } -230 -231 /** @return the total size of the store files in the mob store*/ -232 public long getMobStoreFilesSize() { return hfilesMobSize.get(); } -233 -234 /** @return the total size of the store files in the archive which is not shared -235 * with other snapshots and tables -236 * -237 * This is only calculated when -238 * {@link #getSnapshotStats(Configuration, SnapshotProtos.SnapshotDescription, Map)} -239 * is called with a non-null Map -240 */ -241 public long getNonSharedArchivedStoreFilesSize() { -242 return nonSharedHfilesArchiveSize.get(); -243 } -244 -245 /** @return the percentage of the shared store files */ -246 public float getSharedStoreFilePercentage() { -247 return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100; -248 } -249 -250 /** @return the percentage of the mob store files */ -251 public float getMobStoreFilePercentage() { -252 return ((float) hfilesMobSize.get() / (getStoreFilesSize())) * 100; -253 } -254 -255 /** @return the total log size */ -256 public long getLogsSize() { -257 return logSize.get(); -258 } -259 -260 /** Check if for a give file in archive, if there are other snapshots/tables still -261 * reference it. -262 * @param filePath file path in archive -263 * @param snapshotFilesMap a map for store files in snapshots about how many snapshots refer -264 * to it. -265 * @return true or false -266 */ -267 private boolean isArchivedFileStillReferenced(final Path filePath, -268 final Map<Path, Integer> snapshotFilesMap) { +049import org.apache.hadoop.hbase.util.FSUtils; +050import org.apache.hadoop.util.StringUtils; +051import org.apache.yetus.audience.InterfaceAudience; +052 +053import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +054import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; +055import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; +056 +057/** +058 * Tool for dumping snapshot information. +059 * <ol> +060 * <li> Table Descriptor +061 * <li> Snapshot creation time, type, format version, ... +062 * <li> List of hfiles and wals +063 * <li> Stats about hfiles and logs sizes, percentage of shared with the source table, ... +064 * </ol> +065 */ +066@InterfaceAudience.Public +067public final class SnapshotInfo extends AbstractHBaseTool { +068 private static final Log LOG = LogFactory.getLog(SnapshotInfo.class); +069 +070 static final class Options { +071 static final Option SNAPSHOT = new Option(null, "snapshot", true, "Snapshot to examine."); +072 static final Option REMOTE_DIR = new Option(null, "remote-dir", true, +073 "Root directory that contains the snapshots."); +074 static final Option LIST_SNAPSHOTS = new Option(null, "list-snapshots", false, +075 "List all the available snapshots and exit."); +076 static final Option FILES = new Option(null, "files", false, "Files and logs list."); +077 static final Option STATS = new Option(null, "stats", false, "Files and logs stats."); +078 static final Option SCHEMA = new Option(null, "schema", false, +079 "Describe the snapshotted table."); +080 static final Option SIZE_IN_BYTES = new Option(null, "size-in-bytes", false, +081 "Print the size of the files in bytes."); +082 } +083 +084 /** +085 * Statistics about the snapshot +086 * <ol> +087 * <li> How many store files and logs are in the archive +088 * <li> How many store files and logs are shared with the table +089 * <li> Total store files and logs size and shared amount +090 * </ol> +091 */ +092 public static class SnapshotStats { +093 /** Information about the file referenced by the snapshot */ +094 static class FileInfo { +095 private final boolean corrupted; +096 private final boolean inArchive; +097 private final long size; +098 +099 FileInfo(final boolean inArchive, final long size, final boolean corrupted) { +100 this.corrupted = corrupted; +101 this.inArchive = inArchive; +102 this.size = size; +103 } +104 +105 /** @return true if the file is in the archive */ +106 public boolean inArchive() { +107 return this.inArchive; +108 } +109 +110 /** @return true if the file is corrupted */ +111 public boolean isCorrupted() { +112 return this.corrupted; +113 } +114 +115 /** @return true if the file is missing */ +116 public boolean isMissing() { +117 return this.size < 0; +118 } +119 +120 /** @return the file size */ +121 public long getSize() { +122 return this.size; +123 } +124 +125 String getStateToString() { +126 if (isCorrupted()) return "CORRUPTED"; +127 if (isMissing()) return "NOT FOUND"; +128 if (inArchive()) return "archive"; +129 return null; +130 } +131 } +132 +133 private AtomicInteger hfilesArchiveCount = new AtomicInteger(); +134 private AtomicInteger hfilesCorrupted = new AtomicInteger(); +135 private AtomicInteger hfilesMissing = new AtomicInteger(); +136 private AtomicInteger hfilesCount = new AtomicInteger(); +137 private AtomicInteger hfilesMobCount = new AtomicInteger(); +138 private AtomicInteger logsMissing = new AtomicInteger(); +139 private AtomicInteger logsCount = new AtomicInteger(); +140 private AtomicLong hfilesArchiveSize = new AtomicLong(); +141 private AtomicLong hfilesSize = new AtomicLong(); +142 private AtomicLong hfilesMobSize = new AtomicLong(); +143 private AtomicLong nonSharedHfilesArchiveSize = new AtomicLong(); +144 private AtomicLong logSize = new AtomicLong(); +145 +146 private final SnapshotProtos.SnapshotDescription snapshot; +147 private final TableName snapshotTable; +148 private final Configuration conf; +149 private final FileSystem fs; +150 +151 SnapshotStats(final Configuration conf, final FileSystem fs, +152 final SnapshotDescription snapshot) +153 { +154 this.snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); +155 this.snapshotTable = snapshot.getTableName(); +156 this.conf = conf; +157 this.fs = fs; +158 } +159 +160 SnapshotStats(final Configuration conf, final FileSystem fs, +161 final SnapshotProtos.SnapshotDescription snapshot) { +162 this.snapshot = snapshot; +163 this.snapshotTable = TableName.valueOf(snapshot.getTable()); +164 this.conf = conf; +165 this.fs = fs; +166 } +167 +168 +169 /** @return the snapshot descriptor */ +170 public SnapshotDescription getSnapshotDescription() { +171 return ProtobufUtil.createSnapshotDesc(this.snapshot); +172 } +173 +174 /** @return true if the snapshot is corrupted */ +175 public boolean isSnapshotCorrupted() { +176 return hfilesMissing.get() > 0 || +177 logsMissing.get() > 0 || +178 hfilesCorrupted.get() > 0; +179 } +180 +181 /** @return the number of available store files */ +182 public int getStoreFilesCount() { +183 return hfilesCount.get() + hfilesArchiveCount.get() + hfilesMobCount.get(); +184 } +185 +186 /** @return the number of available store files in the archive */ +187 public int getArchivedStoreFilesCount() { +188 return hfilesArchiveCount.get(); +189 } +190 +191 /** @return the number of available store files in the mob dir */ +192 public int getMobStoreFilesCount() { return hfilesMobCount.get(); } +193 +194 /** @return the number of available log files */ +195 public int getLogsCount() { +196 return logsCount.get(); +197 } +198 +199 /** @return the number of missing store files */ +200 public int getMissingStoreFilesCount() { +201 return hfilesMissing.get(); +202 } +203 +204 /** @return the number of corrupted store files */ +205 public int getCorruptedStoreFilesCount() { +206 return hfilesCorrupted.get(); +207 } +208 +209 /** @return the number of missing log files */ +210 public int getMissingLogsCount() { +211 return logsMissing.get(); +212 } +213 +214 /** @return the total size of the store files referenced by the snapshot */ +215 public long getStoreFilesSize() { +216 return hfilesSize.get() + hfilesArchiveSize.get() + hfilesMobSize.get(); +217 } +218 +219 /** @return the total size of the store files shared */ +220 public long getSharedStoreFilesSize() { +221 return hfilesSize.get(); +222 } +223 +224 /** @return the total size of the store files in the archive */ +225 public long getArchivedStoreFileSize() { +226 return hfilesArchiveSize.get(); +227 } +228 +229 /** @return the total size of the store files in the mob store*/ +230 public long getMobStoreFilesSize() { return hfilesMobSize.get(); } +231 +232 /** @return the total size of the store files in the archive which is not shared +233 * with other snapshots and tables +234 * +235 * This is only calculated when +236 * {@link #getSnapshotStats(Configuration, SnapshotProtos.SnapshotDescription, Map)} +237 * is called with a non-null Map +238 */ +239 public long getNonSharedArchivedStoreFilesSize() { +240 return nonSharedHfilesArchiveSize.get(); +241 } +242 +243 /** @return the percentage of the shared store files */ +244 public float getSharedStoreFilePercentage() { +245 return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100; +246 } +247 +248 /** @return the percentage of the mob store files */ +249 public float getMobStoreFilePercentage() { +250 return ((float) hfilesMobSize.get() / (getStoreFilesSize())) * 100; +251 } +252 +253 /** @return the total log size */ +254 public long getLogsSize() { +255 return logSize.get(); +256 } +257 +258 /** Check if for a give file in archive, if there are other snapshots/tables still +259 * reference it. +260 * @param filePath file path in archive +261 * @param snapshotFilesMap a map for store files in snapshots about how many snapshots refer +262 * to it. +263 * @return true or false +264 */ +265 private boolean isArchivedFileStillReferenced(final Path filePath, +266 final Map<Path, Integer> snapshotFilesMap) { +267 +268 Integer c = snapshotFilesMap.get(filePath); 269 -270 Integer c = snapshotFilesMap.get(filePath); -271 -272 // Check if there are other snapshots or table from clone_snapshot() (via back-reference) -273 // still reference to it. -274 if ((c != null) && (c == 1)) { -275 Path parentDir = filePath.getParent(); -276 Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName()); -277 try { -278 if (FSUtils.listStatus(fs, backRefDir) == null) { -279 return false; -280 } -281 } catch (IOException e) { -282 // For the purpose of this function, IOException is ignored and treated as -283 // the file is still being referenced. -284 } -285 } -286 return true; -287 } -288 -289 /** -290 * Add the specified store file to the stats -291 * @param region region encoded Name -292 * @param family family name -293 * @param storeFile store file name -294 * @param filesMap store files map for all snapshots, it may be null -295 * @return the store file information -296 */ -297 FileInfo addStoreFile(final HRegionInfo region, final String family, -298 final SnapshotRegionManifest.StoreFile storeFile, -299 final Map<Path, Integer> filesMap) throws IOException { -300 HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(), -301 family, storeFile.getName()); -302 boolean isCorrupted = false; -303 boolean inArchive = false; -304 long size = -1; -305 try { -306 if ((inArchive = fs.exists(link.getArchivePath()))) { -307 size = fs.getFileStatus(link.getArchivePath()).getLen(); -308 hfilesArchiveSize.addAndGet(size); -309 hfilesArchiveCount.incrementAndGet(); -310 -311 // If store file is not shared with other snapshots and tables, -312 // increase nonSharedHfilesArchiveSize -313 if ((filesMap != null) && -314 !isArchivedFileStillReferenced(link.getArchivePath(), filesMap)) { -315 nonSharedHfilesArchiveSize.addAndGet(size); -316 } -317 } else if (inArchive = fs.exists(link.getMobPath())) { -318 size = fs.getFileStatus(link.getMobPath()).getLen(); -319 hfilesMobSize.addAndGet(size); -320 hfilesMobCount.incrementAndGet(); -321 } else { -322 size = link.getFileStatus(fs).getLen(); -323 hfilesSize.addAndGet(size); -324 hfilesCount.incrementAndGet(); -325 } -326 isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); -327 if (isCorrupted) hfilesCorrupted.incrementAndGet(); -328 } catch (FileNotFoundException e) { -329 hfilesMissing.incrementAndGet(); -330 } -331 return new FileInfo(inArchive, size, isCorrupted); -332 } -333 -334 /** -335 * Add the specified log file to the stats -336 * @param server server name -337 * @param logfile log file name -338 * @return the log information -339 */ -340 FileInfo addLogFile(final String server, final String logfile) throws IOException { -341 WALLink logLink = new WALLink(conf, server, logfile); -342 long size = -1; -343 try { -344 size = logLink.getFileStatus(fs).getLen(); -345 logSize.addAndGet(size); -346 logsCount.incrementAndGet(); -347 } catch (FileNotFoundException e) { -348 logsMissing.incrementAndGet(); -349 } -350 return new FileInfo(false, size, false); -351 } -352 } -353 -354 private FileSystem fs; -355 private Path rootDir; +270 // Check if there are other snapshots or table from clone_snapshot() (via back-reference) +271 // still reference to it. +272 if ((c != null) && (c == 1)) { +273 Path parentDir = filePath.getParent(); +274 Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName()); +275 try { +276 if (FSUtils.listStatus(fs, backRefDir) == null) { +277 return false; +278 } +279 } catch (IOException e) { +280 // For the purpose of this function, IOException is ignored and treated as +281 // the file is still being referenced. +282 } +283 } +284 return true; +285 } +286 +287 /** +288 * Add the specified store file to the stats +289 * @param region region encoded Name +290 * @param family family name +291 * @param storeFile store file name +292 * @param filesMap store files map for all snapshots, it may be null +293 * @return the store file information +294 */ +295 FileInfo addStoreFile(final RegionInfo region, final String family, +296 final SnapshotRegionManifest.StoreFile storeFile, +297 final Map<Path, Integer> filesMap) throws IOException { +298 HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(), +299 family, storeFile.getName()); +300 boolean isCorrupted = false; +301 boolean inArchive = false; +302 long size = -1; +303 try { +304 if ((inArchive = fs.exists(link.getArchivePath()))) { +305 size = fs.getFileStatus(link.getArchivePath()).getLen(); +306 hfilesArchiveSize.addAndGet(size); +307 hfilesArchiveCount.incrementAndGet(); +308 +309 // If store file is not shared with other snapshots and tables, +310 // increase nonSharedHfilesArchiveSize +311 if ((filesMap != null) && +312 !isArchivedFileStillReferenced(link.getArchivePath(), filesMap)) { +313 nonSharedHfilesArchiveSize.addAndGet(size); +314 } +315 } else if (inArchive = fs.exists(link.getMobPath())) { +316 size = fs.getFileStatus(link.getMobPath()).getLen(); +317 hfilesMobSize.addAndGet(size); +318 hfilesMobCount.incrementAndGet(); +319 } else { +320 size = link.getFileStatus(fs).getLen(); +321 hfilesSize.addAndGet(size); +322 hfilesCount.incrementAndGet(); +323 } +324 isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); +325 if (isCorrupted) hfilesCorrupted.incrementAndGet(); +326 } catch (FileNotFoundException e) { +327 hfilesMissing.incrementAndGet(); +328 } +329 return new FileInfo(inArchive, size, isCorrupted); +330 } +331 +332 /** +333 * Add the specified log file to the stats +334 * @param server server name +335 * @param logfile log file name +336 * @return the log information +337 */ +338 FileInfo addLogFile(final String server, final String logfile) throws IOException { +339 WALLink logLink = new WALLink(conf, server, logfile); +340 long size = -1; +341 try { +342 size = logLink.getFileStatus(fs).getLen(); +343 logSize.addAndGet(size); +344 logsCount.incrementAndGet(); +345 } catch (FileNotFoundException e) { +346 logsMissing.incrementAndGet(); +347 } +348 return new FileInfo(false, size, false); +349 } +350 } +351 +352 private FileSystem fs; +353 private Path rootDir; +354 +355 private SnapshotManifest snapshotManifest; 356 -357 private SnapshotManifest snapshotManifest; -358 -359 private boolean listSnapshots = false; -360 private String snapshotName; -361 private Path remoteDir; -362 private boolean showSchema = false; -363 private boolean showFiles = false; -364 private boolean showStats = false; -365 private boolean printSizeInBytes = false; -366 -367 @Override -368 public int doWork() throws IOException, InterruptedException { -369 if (remoteDir != null) { -370 URI defaultFs = remoteDir.getFileSystem(conf).getUri(); -371 FSUtils.setFsDefault(conf, new Path(defaultFs)); -372 FSUtils.setRootDir(conf, remoteDir); -373 } -374 -375 // List Available Snapshots -376 if (listSnapshots) { -377 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); -378 System.out.printf("%-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TABLE NAME"); -379 for (SnapshotDescription desc: getSnapshotList(conf)) { -380 System.out.printf("%-20s | %20s | %s%n", -381 desc.getName(), -382 df.format(new Date(desc.getCreationTime())), -383 desc.getTableNameAsString()); -384 } -385 return 0; -386 } -387 -388 rootDir = FSUtils.getRootDir(conf); -389 fs = FileSystem.get(rootDir.toUri(), conf); -390 LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir); -391 -392 // Load snapshot information -393 if (!loadSnapshotInfo(snapshotName)) { -394 System.err.println("Snapshot '" + snapshotName + "' not found!"); -395 return 1; -396 } -397 -398 printInfo(); -399 if (showSchema) printSchema(); -400 printFiles(showFiles, showStats); -401 -402 return 0; -403 } -404 -405 /** -406 * Load snapshot info and table descriptor for the specified snapshot -407 * @param snapshotName name of the snapshot to load -408 * @return false if snapshot is not found -409 */ -410 private boolean loadSnapshotInfo(final String snapshotName) throws IOException { -411 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); -412 if (!fs.exists(snapshotDir)) { -413 LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir); -414 return false; -415 } -416 -417 SnapshotProtos.SnapshotDescription snapshotDesc = -418 SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); -419 snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc); -420 return true; -421 } -422 -423 /** -424 * Dump the {@link SnapshotDescription} -425 */ -426 private void printInfo() { -427 SnapshotProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); -428 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); -429 System.out.println("Snapshot Info"); -430 System.out.println("----------------------------------------"); -431 System.out.println(" Name: " + snapshotDesc.getName()); -432 System.out.println(" Type: " + snapshotDesc.getType()); -433 System.out.println(" Table: " + snapshotDesc.getTable()); -434 System.out.println(" Format: " + snapshotDesc.getVersion()); -435 System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime()))); -436 System.out.println(" Owner: " + snapshotDesc.getOwner()); -437 System.out.println(); -438 } -439 -440 /** -441 * Dump the {@link org.apache.hadoop.hbase.client.TableDescriptor} -442 */ -443 private void printSchema() { -444 System.out.println("Table Descriptor"); -445 System.out.println("----------------------------------------"); -446 System.out.println(snapshotManifest.getTableDescriptor().toString()); -447 System.out.println(); -448 } -449 -450 /** -451 * Collect the hfiles and logs statistics of the snapshot and -452 * dump the file list if requested and the collected information. -453 */ -454 private void printFiles(final boolean showFiles, final boolean showStats) throws IOException { -455 if (showFiles) { -456 System.out.println("Snapshot Files"); -457 System.out.println("----------------------------------------"); -458 } -459 -460 // Collect information about hfiles and logs in the snapshot -461 final SnapshotProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); -462 final String table = snapshotDesc.getTable(); -463 final SnapshotDescription desc = ProtobufUtil.createSnapshotDesc(snapshotDesc); -464 final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc); -465 SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest, -466 "SnapshotInfo", -467 new SnapshotReferenceUtil.SnapshotVisitor() { -468 @Override -469 public void storeFile(final HRegionInfo regionInfo, final String family, -470 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { -471 if (storeFile.hasReference()) return; -472 -473 SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile, null); -474 if (showFiles) { -475 String state = info.getStateToString(); -476 System.out.printf("%8s %s/%s/%s/%s %s%n", -477 (info.isMissing() ? "-" : fileSizeToString(info.getSize())), -478 table, regionInfo.getEncodedName(), family, storeFile.getName(), -479 state == null ? "" : "(" + state + ")"); -480 } -481 } -482 }); -483 -484 // Dump the stats -485 System.out.println(); -486 if (stats.isSnapshotCorrupted()) { -487 System.out.println("**************************************************************"); -488 System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n", -489 stats.getMissingStoreFilesCount(), stats.getMissingLogsCount()); -490 System.out.printf(" %d hfile(s) corrupted.%n", -491 stats.getCorruptedStoreFilesCount()); -492 System.out.println("**************************************************************"); -493 } -494 -495 if (showStats) { -496 System.out.printf("%d HFiles (%d in archive, %d in mob storage), total size %s " + -497 "(%.2f%% %s shared with the source table, %.2f%% %s in mob dir)%n", -498 stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), -499 stats.getMobStoreFilesCount(), -500 fileSizeToString(stats.getStoreFilesSize()), -501 stats.getSharedStoreFilePercentage(), -502 fileSizeToString(stats.getSharedStoreFilesSize()), -503 stats.getMobStoreFilePercentage(), -504 fileSizeToString(stats.getMobStoreFilesSize()) -505 ); -506 System.out.printf("%d Logs, total size %s%n", -507 stats.getLogsCount(), fileSizeToString(stats.getLogsSize())); -508 System.out.println(); -509 } -510 } -511 -512 private String fileSizeToString(long size) { -513 return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size); -514 } -515 -516 @Override -517 protected void addOptions() { -518 addRequiredOption(Options.SNAPSHOT); -519 addOption(Options.REMOTE_DIR); -520 addOption(Options.LIST_SNAPSHOTS); -521 addOption(Options.FILES); -522 addOption(Options.STATS); -523 addOption(Options.SCHEMA); -524 addOption(Options.SIZE_IN_BYTES); -525 } -526 -527 @Override -528 protected void processOptions(CommandLine cmd) { -529 snapshotName = cmd.getOptionValue(Options.SNAPSHOT.getLongOpt()); -530 showFiles = cmd.hasOption(Options.FILES.getLongOpt()); -531 showStats = cmd.hasOption(Options.FILES.getLongOpt()) -532 || cmd.hasOption(Options.STATS.getLongOpt()); -533 showSchema = cmd.hasOption(Options.SCHEMA.getLongOpt()); -534 listSnapshots = cmd.hasOption(Options.LIST_SNAPSHOTS.getLongOpt()); -535 printSizeInBytes = cmd.hasOption(Options.SIZE_IN_BYTES.getLongOpt()); -536 if (cmd.hasOption(Options.REMOTE_DIR.getLongOpt())) { -537 remoteDir = new Path(cmd.getOptionValue(Options.REMOTE_DIR.getLongOpt())); -538 } -539 } -540 -541 @Override -542 protected void printUsage() { -543 printUsage("hbase snapshot info [options]", "Options:", ""); -544 System.err.println("Examples:"); -545 System.err.println(" hbase snapshot info --snapshot MySnapshot --files"); -546 } -547 -548 /** -549 * Returns the snapshot stats -550 * @param conf the {@link Configuration} to use -551 * @param snapshot {@link SnapshotDescription} to get stats from -552 * @return the snapshot stats -553 */ -554 public static SnapshotStats getSnapshotStats(final Configuration conf, -555 final SnapshotDescription snapshot) throws IOException { -556 SnapshotProtos.SnapshotDescription snapshotDesc = -557 ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); -558 return getSnapshotStats(conf, snapshotDesc, null); -559 } -560 -561 /** -562 * Returns the snapshot stats -563 * @param conf the {@link Configuration} to use -564 * @param snapshotDesc HBaseProtos.SnapshotDescription to get stats from -565 * @param filesMap {@link Map} store files map for all snapshots, it may be null -566 * @return the snapshot stats -567 */ -568 public static SnapshotStats getSnapshotStats(final Configuration conf, -569 final SnapshotProtos.SnapshotDescription snapshotDesc, -570 final Map<Path, Integer> filesMap) throws IOException { -571 Path rootDir = FSUtils.getRootDir(conf); -572 FileSystem fs = FileSystem.get(rootDir.toUri(), conf); -573 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir); -574 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); -575 final SnapshotStats stats = new SnapshotStats(conf, fs, snapshotDesc); -576 SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, -577 "SnapshotsStatsAggregation", new SnapshotReferenceUtil.SnapshotVisitor() { -578 @Override -579 public void storeFile(final HRegionInfo regionInfo, final String family, -580 final SnapshotRegionManifest.StoreFile storeFile) throws IOException { -581 if (!storeFile.hasReference()) { -582 stats.addStoreFile(regionInfo, family, storeFile, filesMap); -583 } -584 }}); -585 return stats; -586 } -587 -588 /** -589 * Returns the list of available snapshots in the specified location -590 * @param conf the {@link Configuration} to use -591 * @return the list of snapshots -592 */ -593 public static List<SnapshotDescription> getSnapshotList(final Configuration conf) -594 throws IOException { -595 Path rootDir = FSUtils.getRootDir(conf); -596 FileSystem fs = FileSystem.get(rootDir.toUri(), conf); -597 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); -598 FileStatus[] snapshots = fs.listStatus(snapshotDir, -599 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); -600 List<SnapshotDescription> snapshotLists = new ArrayList<>(snapshots.length); -601 for (FileStatus snapshotDirStat: snapshots) { -602 SnapshotProtos.SnapshotDescription snapshotDesc = -603 SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()); -604 snapshotLists.add(ProtobufUtil.createSnapshotDesc(snapshotDesc)); -605 } -606 return snapshotLists; -607