Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 956351866B for ; Wed, 16 Dec 2015 16:50:56 +0000 (UTC) Received: (qmail 36179 invoked by uid 500); 16 Dec 2015 16:50:48 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 36038 invoked by uid 500); 16 Dec 2015 16:50:47 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 34548 invoked by uid 99); 16 Dec 2015 16:50:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Dec 2015 16:50:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E9724E09B2; Wed, 16 Dec 2015 16:50:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Wed, 16 Dec 2015 16:51:12 -0000 Message-Id: <6e9c4325e634410aa9156349e82c9982@git.apache.org> In-Reply-To: <80cf44be04ef48d5b3105b20c6230b5c@git.apache.org> References: <80cf44be04ef48d5b3105b20c6230b5c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [27/51] [partial] hbase-site git commit: Published site at 60d33ce34191533bb858852584bd9bddfeb16a23. http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539ad177/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.KeyValueStatsCollector.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.KeyValueStatsCollector.html b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.KeyValueStatsCollector.html index b065ec0..519b3ef 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.KeyValueStatsCollector.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.KeyValueStatsCollector.html @@ -31,633 +31,748 @@ 023import java.io.DataInput; 024import java.io.IOException; 025import java.io.PrintStream; -026import java.util.ArrayList; -027import java.util.HashMap; -028import java.util.Iterator; -029import java.util.LinkedHashSet; -030import java.util.List; -031import java.util.Locale; -032import java.util.Map; -033import java.util.Set; -034import java.util.SortedMap; -035 -036import org.apache.commons.cli.CommandLine; -037import org.apache.commons.cli.CommandLineParser; -038import org.apache.commons.cli.HelpFormatter; -039import org.apache.commons.cli.Option; -040import org.apache.commons.cli.OptionGroup; -041import org.apache.commons.cli.Options; -042import org.apache.commons.cli.ParseException; -043import org.apache.commons.cli.PosixParser; -044import org.apache.commons.logging.Log; -045import org.apache.commons.logging.LogFactory; -046import org.apache.hadoop.hbase.classification.InterfaceAudience; -047import org.apache.hadoop.hbase.classification.InterfaceStability; -048import org.apache.hadoop.conf.Configuration; -049import org.apache.hadoop.conf.Configured; -050import org.apache.hadoop.fs.FileSystem; -051import org.apache.hadoop.fs.Path; -052import org.apache.hadoop.hbase.Cell; -053import org.apache.hadoop.hbase.CellComparator; -054import org.apache.hadoop.hbase.CellUtil; -055import org.apache.hadoop.hbase.HBaseInterfaceAudience; -056import org.apache.hadoop.hbase.HConstants; -057import org.apache.hadoop.hbase.TableName; -058import org.apache.hadoop.hbase.HBaseConfiguration; -059import org.apache.hadoop.hbase.HRegionInfo; -060import org.apache.hadoop.hbase.KeyValueUtil; -061import org.apache.hadoop.hbase.Tag; -062import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; -063import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; -064import org.apache.hadoop.hbase.mob.MobUtils; -065import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; -066import org.apache.hadoop.hbase.util.BloomFilter; -067import org.apache.hadoop.hbase.util.BloomFilterUtil; -068import org.apache.hadoop.hbase.util.BloomFilterFactory; -069import org.apache.hadoop.hbase.util.Bytes; -070import org.apache.hadoop.hbase.util.FSUtils; -071import org.apache.hadoop.hbase.util.HFileArchiveUtil; -072import org.apache.hadoop.hbase.util.Writables; -073import org.apache.hadoop.util.Tool; -074import org.apache.hadoop.util.ToolRunner; -075 -076import com.yammer.metrics.core.Histogram; -077import com.yammer.metrics.core.Metric; -078import com.yammer.metrics.core.MetricName; -079import com.yammer.metrics.core.MetricPredicate; -080import com.yammer.metrics.core.MetricsRegistry; -081import com.yammer.metrics.reporting.ConsoleReporter; -082 -083/** -084 * Implements pretty-printing functionality for {@link HFile}s. -085 */ -086@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) -087@InterfaceStability.Evolving -088public class HFilePrettyPrinter extends Configured implements Tool { -089 -090 private static final Log LOG = LogFactory.getLog(HFilePrettyPrinter.class); -091 -092 private Options options = new Options(); -093 -094 private boolean verbose; -095 private boolean printValue; -096 private boolean printKey; -097 private boolean shouldPrintMeta; -098 private boolean printBlockIndex; -099 private boolean printBlockHeaders; -100 private boolean printStats; -101 private boolean checkRow; -102 private boolean checkFamily; -103 private boolean isSeekToRow = false; -104 private boolean checkMobIntegrity = false; -105 private Map<String, List<Path>> mobFileLocations; -106 private static final int FOUND_MOB_FILES_CACHE_CAPACITY = 50; -107 private static final int MISSING_MOB_FILES_CACHE_CAPACITY = 20; -108 -109 /** -110 * The row which the user wants to specify and print all the KeyValues for. -111 */ -112 private byte[] row = null; -113 -114 private List<Path> files = new ArrayList<Path>(); -115 private int count; -116 -117 private static final String FOUR_SPACES = " "; +026import java.text.DateFormat; +027import java.util.ArrayList; +028import java.util.HashMap; +029import java.util.Iterator; +030import java.util.LinkedHashSet; +031import java.util.List; +032import java.util.Locale; +033import java.util.Map; +034import java.util.Set; +035import java.util.SortedMap; +036import java.util.TimeZone; +037import java.util.concurrent.TimeUnit; +038 +039import org.apache.commons.cli.CommandLine; +040import org.apache.commons.cli.CommandLineParser; +041import org.apache.commons.cli.HelpFormatter; +042import org.apache.commons.cli.Option; +043import org.apache.commons.cli.OptionGroup; +044import org.apache.commons.cli.Options; +045import org.apache.commons.cli.ParseException; +046import org.apache.commons.cli.PosixParser; +047import org.apache.commons.lang.StringUtils; +048import org.apache.commons.logging.Log; +049import org.apache.commons.logging.LogFactory; +050import org.apache.hadoop.hbase.classification.InterfaceAudience; +051import org.apache.hadoop.hbase.classification.InterfaceStability; +052import org.apache.hadoop.conf.Configuration; +053import org.apache.hadoop.conf.Configured; +054import org.apache.hadoop.fs.FileSystem; +055import org.apache.hadoop.fs.Path; +056import org.apache.hadoop.hbase.Cell; +057import org.apache.hadoop.hbase.CellComparator; +058import org.apache.hadoop.hbase.CellUtil; +059import org.apache.hadoop.hbase.HBaseInterfaceAudience; +060import org.apache.hadoop.hbase.HConstants; +061import org.apache.hadoop.hbase.TableName; +062import org.apache.hadoop.hbase.HBaseConfiguration; +063import org.apache.hadoop.hbase.HRegionInfo; +064import org.apache.hadoop.hbase.KeyValueUtil; +065import org.apache.hadoop.hbase.Tag; +066import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +067import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; +068import org.apache.hadoop.hbase.mob.MobUtils; +069import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; +070import org.apache.hadoop.hbase.util.BloomFilter; +071import org.apache.hadoop.hbase.util.BloomFilterUtil; +072import org.apache.hadoop.hbase.util.BloomFilterFactory; +073import org.apache.hadoop.hbase.util.Bytes; +074import org.apache.hadoop.hbase.util.FSUtils; +075import org.apache.hadoop.hbase.util.HFileArchiveUtil; +076import org.apache.hadoop.hbase.util.Writables; +077import org.apache.hadoop.util.Tool; +078import org.apache.hadoop.util.ToolRunner; +079 +080import com.codahale.metrics.Histogram; +081import com.codahale.metrics.Counter; +082import com.codahale.metrics.Gauge; +083import com.codahale.metrics.Meter; +084import com.codahale.metrics.MetricFilter; +085import com.codahale.metrics.MetricRegistry; +086import com.codahale.metrics.ConsoleReporter; +087import com.codahale.metrics.ScheduledReporter; +088import com.codahale.metrics.Snapshot; +089import com.codahale.metrics.Timer; +090 +091import static com.codahale.metrics.MetricRegistry.name; +092 +093/** +094 * Implements pretty-printing functionality for {@link HFile}s. +095 */ +096@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) +097@InterfaceStability.Evolving +098public class HFilePrettyPrinter extends Configured implements Tool { +099 +100 private static final Log LOG = LogFactory.getLog(HFilePrettyPrinter.class); +101 +102 private Options options = new Options(); +103 +104 private boolean verbose; +105 private boolean printValue; +106 private boolean printKey; +107 private boolean shouldPrintMeta; +108 private boolean printBlockIndex; +109 private boolean printBlockHeaders; +110 private boolean printStats; +111 private boolean checkRow; +112 private boolean checkFamily; +113 private boolean isSeekToRow = false; +114 private boolean checkMobIntegrity = false; +115 private Map<String, List<Path>> mobFileLocations; +116 private static final int FOUND_MOB_FILES_CACHE_CAPACITY = 50; +117 private static final int MISSING_MOB_FILES_CACHE_CAPACITY = 20; 118 -119 public HFilePrettyPrinter() { -120 super(); -121 init(); -122 } +119 /** +120 * The row which the user wants to specify and print all the KeyValues for. +121 */ +122 private byte[] row = null; 123 -124 public HFilePrettyPrinter(Configuration conf) { -125 super(conf); -126 init(); -127 } +124 private List<Path> files = new ArrayList<Path>(); +125 private int count; +126 +127 private static final String FOUR_SPACES = " "; 128 -129 private void init() { -130 options.addOption("v", "verbose", false, -131 "Verbose output; emits file and meta data delimiters"); -132 options.addOption("p", "printkv", false, "Print key/value pairs"); -133 options.addOption("e", "printkey", false, "Print keys"); -134 options.addOption("m", "printmeta", false, "Print meta data of file"); -135 options.addOption("b", "printblocks", false, "Print block index meta data"); -136 options.addOption("h", "printblockheaders", false, "Print block headers for each block."); -137 options.addOption("k", "checkrow", false, -138 "Enable row order check; looks for out-of-order keys"); -139 options.addOption("a", "checkfamily", false, "Enable family check"); -140 options.addOption("w", "seekToRow", true, -141 "Seek to this row and print all the kvs for this row only"); -142 options.addOption("s", "stats", false, "Print statistics"); -143 options.addOption("i", "checkMobIntegrity", false, -144 "Print all cells whose mob files are missing"); -145 -146 OptionGroup files = new OptionGroup(); -147 files.addOption(new Option("f", "file", true, -148 "File to scan. Pass full-path; e.g. hdfs://a:9000/hbase/hbase:meta/12/34")); -149 files.addOption(new Option("r", "region", true, -150 "Region to scan. Pass region name; e.g. 'hbase:meta,,1'")); -151 options.addOptionGroup(files); -152 } -153 -154 public boolean parseOptions(String args[]) throws ParseException, -155 IOException { -156 if (args.length == 0) { -157 HelpFormatter formatter = new HelpFormatter(); -158 formatter.printHelp("HFile", options, true); -159 return false; -160 } -161 CommandLineParser parser = new PosixParser(); -162 CommandLine cmd = parser.parse(options, args); +129 public HFilePrettyPrinter() { +130 super(); +131 init(); +132 } +133 +134 public HFilePrettyPrinter(Configuration conf) { +135 super(conf); +136 init(); +137 } +138 +139 private void init() { +140 options.addOption("v", "verbose", false, +141 "Verbose output; emits file and meta data delimiters"); +142 options.addOption("p", "printkv", false, "Print key/value pairs"); +143 options.addOption("e", "printkey", false, "Print keys"); +144 options.addOption("m", "printmeta", false, "Print meta data of file"); +145 options.addOption("b", "printblocks", false, "Print block index meta data"); +146 options.addOption("h", "printblockheaders", false, "Print block headers for each block."); +147 options.addOption("k", "checkrow", false, +148 "Enable row order check; looks for out-of-order keys"); +149 options.addOption("a", "checkfamily", false, "Enable family check"); +150 options.addOption("w", "seekToRow", true, +151 "Seek to this row and print all the kvs for this row only"); +152 options.addOption("s", "stats", false, "Print statistics"); +153 options.addOption("i", "checkMobIntegrity", false, +154 "Print all cells whose mob files are missing"); +155 +156 OptionGroup files = new OptionGroup(); +157 files.addOption(new Option("f", "file", true, +158 "File to scan. Pass full-path; e.g. hdfs://a:9000/hbase/hbase:meta/12/34")); +159 files.addOption(new Option("r", "region", true, +160 "Region to scan. Pass region name; e.g. 'hbase:meta,,1'")); +161 options.addOptionGroup(files); +162 } 163 -164 verbose = cmd.hasOption("v"); -165 printValue = cmd.hasOption("p"); -166 printKey = cmd.hasOption("e") || printValue; -167 shouldPrintMeta = cmd.hasOption("m"); -168 printBlockIndex = cmd.hasOption("b"); -169 printBlockHeaders = cmd.hasOption("h"); -170 printStats = cmd.hasOption("s"); -171 checkRow = cmd.hasOption("k"); -172 checkFamily = cmd.hasOption("a"); -173 checkMobIntegrity = cmd.hasOption("i"); -174 -175 if (cmd.hasOption("f")) { -176 files.add(new Path(cmd.getOptionValue("f"))); -177 } -178 -179 if (cmd.hasOption("w")) { -180 String key = cmd.getOptionValue("w"); -181 if (key != null && key.length() != 0) { -182 row = Bytes.toBytesBinary(key); -183 isSeekToRow = true; -184 } else { -185 System.err.println("Invalid row is specified."); -186 System.exit(-1); -187 } -188 } -189 -190 if (cmd.hasOption("r")) { -191 String regionName = cmd.getOptionValue("r"); -192 byte[] rn = Bytes.toBytes(regionName); -193 byte[][] hri = HRegionInfo.parseRegionName(rn); -194 Path rootDir = FSUtils.getRootDir(getConf()); -195 Path tableDir = FSUtils.getTableDir(rootDir, TableName.valueOf(hri[0])); -196 String enc = HRegionInfo.encodeRegionName(rn); -197 Path regionDir = new Path(tableDir, enc); -198 if (verbose) -199 System.out.println("region dir -> " + regionDir); -200 List<Path> regionFiles = HFile.getStoreFiles(FileSystem.get(getConf()), -201 regionDir); -202 if (verbose) -203 System.out.println("Number of region files found -> " -204 + regionFiles.size()); -205 if (verbose) { -206 int i = 1; -207 for (Path p : regionFiles) { -208 if (verbose) -209 System.out.println("Found file[" + i++ + "] -> " + p); -210 } -211 } -212 files.addAll(regionFiles); -213 } -214 -215 if(checkMobIntegrity) { -216 if (verbose) { -217 System.out.println("checkMobIntegrity is enabled"); -218 } -219 mobFileLocations = new HashMap<String, List<Path>>(); -220 } -221 return true; -222 } -223 -224 /** -225 * Runs the command-line pretty-printer, and returns the desired command -226 * exit code (zero for success, non-zero for failure). -227 */ -228 @Override -229 public int run(String[] args) { -230 if (getConf() == null) { -231 throw new RuntimeException("A Configuration instance must be provided."); -232 } -233 try { -234 FSUtils.setFsDefault(getConf(), FSUtils.getRootDir(getConf())); -235 if (!parseOptions(args)) -236 return 1; -237 } catch (IOException ex) { -238 LOG.error("Error parsing command-line options", ex); -239 return 1; -240 } catch (ParseException ex) { -241 LOG.error("Error parsing command-line options", ex); -242 return 1; -243 } -244 -245 // iterate over all files found -246 for (Path fileName : files) { -247 try { -248 processFile(fileName); -249 } catch (IOException ex) { -250 LOG.error("Error reading " + fileName, ex); -251 System.exit(-2); -252 } +164 public boolean parseOptions(String args[]) throws ParseException, +165 IOException { +166 if (args.length == 0) { +167 HelpFormatter formatter = new HelpFormatter(); +168 formatter.printHelp("HFile", options, true); +169 return false; +170 } +171 CommandLineParser parser = new PosixParser(); +172 CommandLine cmd = parser.parse(options, args); +173 +174 verbose = cmd.hasOption("v"); +175 printValue = cmd.hasOption("p"); +176 printKey = cmd.hasOption("e") || printValue; +177 shouldPrintMeta = cmd.hasOption("m"); +178 printBlockIndex = cmd.hasOption("b"); +179 printBlockHeaders = cmd.hasOption("h"); +180 printStats = cmd.hasOption("s"); +181 checkRow = cmd.hasOption("k"); +182 checkFamily = cmd.hasOption("a"); +183 checkMobIntegrity = cmd.hasOption("i"); +184 +185 if (cmd.hasOption("f")) { +186 files.add(new Path(cmd.getOptionValue("f"))); +187 } +188 +189 if (cmd.hasOption("w")) { +190 String key = cmd.getOptionValue("w"); +191 if (key != null && key.length() != 0) { +192 row = Bytes.toBytesBinary(key); +193 isSeekToRow = true; +194 } else { +195 System.err.println("Invalid row is specified."); +196 System.exit(-1); +197 } +198 } +199 +200 if (cmd.hasOption("r")) { +201 String regionName = cmd.getOptionValue("r"); +202 byte[] rn = Bytes.toBytes(regionName); +203 byte[][] hri = HRegionInfo.parseRegionName(rn); +204 Path rootDir = FSUtils.getRootDir(getConf()); +205 Path tableDir = FSUtils.getTableDir(rootDir, TableName.valueOf(hri[0])); +206 String enc = HRegionInfo.encodeRegionName(rn); +207 Path regionDir = new Path(tableDir, enc); +208 if (verbose) +209 System.out.println("region dir -> " + regionDir); +210 List<Path> regionFiles = HFile.getStoreFiles(FileSystem.get(getConf()), +211 regionDir); +212 if (verbose) +213 System.out.println("Number of region files found -> " +214 + regionFiles.size()); +215 if (verbose) { +216 int i = 1; +217 for (Path p : regionFiles) { +218 if (verbose) +219 System.out.println("Found file[" + i++ + "] -> " + p); +220 } +221 } +222 files.addAll(regionFiles); +223 } +224 +225 if(checkMobIntegrity) { +226 if (verbose) { +227 System.out.println("checkMobIntegrity is enabled"); +228 } +229 mobFileLocations = new HashMap<String, List<Path>>(); +230 } +231 return true; +232 } +233 +234 /** +235 * Runs the command-line pretty-printer, and returns the desired command +236 * exit code (zero for success, non-zero for failure). +237 */ +238 @Override +239 public int run(String[] args) { +240 if (getConf() == null) { +241 throw new RuntimeException("A Configuration instance must be provided."); +242 } +243 try { +244 FSUtils.setFsDefault(getConf(), FSUtils.getRootDir(getConf())); +245 if (!parseOptions(args)) +246 return 1; +247 } catch (IOException ex) { +248 LOG.error("Error parsing command-line options", ex); +249 return 1; +250 } catch (ParseException ex) { +251 LOG.error("Error parsing command-line options", ex); +252 return 1; 253 } 254 -255 if (verbose || printKey) { -256 System.out.println("Scanned kv count -> " + count); -257 } -258 -259 return 0; -260 } -261 -262 private void processFile(Path file) throws IOException { -263 if (verbose) -264 System.out.println("Scanning -> " + file); -265 FileSystem fs = file.getFileSystem(getConf()); -266 if (!fs.exists(file)) { -267 System.err.println("ERROR, file doesnt exist: " + file); -268 System.exit(-2); -269 } -270 -271 HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(getConf()), getConf()); -272 -273 Map<byte[], byte[]> fileInfo = reader.loadFileInfo(); -274 -275 KeyValueStatsCollector fileStats = null; -276 -277 if (verbose || printKey || checkRow || checkFamily || printStats || checkMobIntegrity) { -278 // scan over file and read key/value's and check if requested -279 HFileScanner scanner = reader.getScanner(false, false, false); -280 fileStats = new KeyValueStatsCollector(); -281 boolean shouldScanKeysValues = false; -282 if (this.isSeekToRow) { -283 // seek to the first kv on this row -284 shouldScanKeysValues = -285 (scanner.seekTo(KeyValueUtil.createFirstOnRow(this.row)) != -1); -286 } else { -287 shouldScanKeysValues = scanner.seekTo(); -288 } -289 if (shouldScanKeysValues) -290 scanKeysValues(file, fileStats, scanner, row); -291 } -292 -293 // print meta data -294 if (shouldPrintMeta) { -295 printMeta(reader, fileInfo); -296 } -297 -298 if (printBlockIndex) { -299 System.out.println("Block Index:"); -300 System.out.println(reader.getDataBlockIndexReader()); +255 // iterate over all files found +256 for (Path fileName : files) { +257 try { +258 processFile(fileName); +259 } catch (IOException ex) { +260 LOG.error("Error reading " + fileName, ex); +261 System.exit(-2); +262 } +263 } +264 +265 if (verbose || printKey) { +266 System.out.println("Scanned kv count -> " + count); +267 } +268 +269 return 0; +270 } +271 +272 private void processFile(Path file) throws IOException { +273 if (verbose) +274 System.out.println("Scanning -> " + file); +275 FileSystem fs = file.getFileSystem(getConf()); +276 if (!fs.exists(file)) { +277 System.err.println("ERROR, file doesnt exist: " + file); +278 System.exit(-2); +279 } +280 +281 HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(getConf()), getConf()); +282 +283 Map<byte[], byte[]> fileInfo = reader.loadFileInfo(); +284 +285 KeyValueStatsCollector fileStats = null; +286 +287 if (verbose || printKey || checkRow || checkFamily || printStats || checkMobIntegrity) { +288 // scan over file and read key/value's and check if requested +289 HFileScanner scanner = reader.getScanner(false, false, false); +290 fileStats = new KeyValueStatsCollector(); +291 boolean shouldScanKeysValues = false; +292 if (this.isSeekToRow) { +293 // seek to the first kv on this row +294 shouldScanKeysValues = +295 (scanner.seekTo(KeyValueUtil.createFirstOnRow(this.row)) != -1); +296 } else { +297 shouldScanKeysValues = scanner.seekTo(); +298 } +299 if (shouldScanKeysValues) +300 scanKeysValues(file, fileStats, scanner, row); 301 } 302 -303 if (printBlockHeaders) { -304 System.out.println("Block Headers:"); -305 /* -306 * TODO: this same/similar block iteration logic is used in HFileBlock#blockRange and -307 * TestLazyDataBlockDecompression. Refactor? -308 */ -309 FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, file); -310 long fileSize = fs.getFileStatus(file).getLen(); -311 FixedFileTrailer trailer = -312 FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize); -313 long offset = trailer.getFirstDataBlockOffset(), -314 max = trailer.getLastDataBlockOffset(); -315 HFileBlock block; -316 while (offset <= max) { -317 block = reader.readBlock(offset, -1, /* cacheBlock */ false, /* pread */ false, -318 /* isCompaction */ false, /* updateCacheMetrics */ false, null, null); -319 offset += block.getOnDiskSizeWithHeader(); -320 System.out.println(block); -321 } -322 } -323 -324 if (printStats) { -325 fileStats.finish(); -326 System.out.println("Stats:\n" + fileStats); -327 } -328 -329 reader.close(); -330 } -331 -332 private void scanKeysValues(Path file, KeyValueStatsCollector fileStats, -333 HFileScanner scanner, byte[] row) throws IOException { -334 Cell pCell = null; -335 FileSystem fs = FileSystem.get(getConf()); -336 Set<String> foundMobFiles = new LinkedHashSet<String>(FOUND_MOB_FILES_CACHE_CAPACITY); -337 Set<String> missingMobFiles = new LinkedHashSet<String>(MISSING_MOB_FILES_CACHE_CAPACITY); -338 do { -339 Cell cell = scanner.getCell(); -340 if (row != null && row.length != 0) { -341 int result = CellComparator.COMPARATOR.compareRows(cell, row, 0, row.length); -342 if (result > 0) { -343 break; -344 } else if (result < 0) { -345 continue; -346 } -347 } -348 // collect stats -349 if (printStats) { -350 fileStats.collect(cell); -351 } -352 // dump key value -353 if (printKey) { -354 System.out.print("K: " + cell); -355 if (printValue) { -356 System.out.print(" V: " -357 + Bytes.toStringBinary(cell.getValueArray(), cell.getValueOffset(), -358 cell.getValueLength())); -359 int i = 0; -360 List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), -361 cell.getTagsLength()); -362 for (Tag tag : tags) { -363 System.out.print(String.format(" T[%d]: %s", i++, -364 Bytes.toStringBinary(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength()))); -365 } -366 } -367 System.out.println(); -368 } -369 // check if rows are in order -370 if (checkRow && pCell != null) { -371 if (CellComparator.COMPARATOR.compareRows(pCell, cell) > 0) { -372 System.err.println("WARNING, previous row is greater then" -373 + " current row\n\tfilename -> " + file + "\n\tprevious -> " -374 + CellUtil.getCellKeyAsString(pCell) + "\n\tcurrent -> " -375 + CellUtil.getCellKeyAsString(cell)); +303 // print meta data +304 if (shouldPrintMeta) { +305 printMeta(reader, fileInfo); +306 } +307 +308 if (printBlockIndex) { +309 System.out.println("Block Index:"); +310 System.out.println(reader.getDataBlockIndexReader()); +311 } +312 +313 if (printBlockHeaders) { +314 System.out.println("Block Headers:"); +315 /* +316 * TODO: this same/similar block iteration logic is used in HFileBlock#blockRange and +317 * TestLazyDataBlockDecompression. Refactor? +318 */ +319 FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, file); +320 long fileSize = fs.getFileStatus(file).getLen(); +321 FixedFileTrailer trailer = +322 FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize); +323 long offset = trailer.getFirstDataBlockOffset(), +324 max = trailer.getLastDataBlockOffset(); +325 HFileBlock block; +326 while (offset <= max) { +327 block = reader.readBlock(offset, -1, /* cacheBlock */ false, /* pread */ false, +328 /* isCompaction */ false, /* updateCacheMetrics */ false, null, null); +329 offset += block.getOnDiskSizeWithHeader(); +330 System.out.println(block); +331 } +332 } +333 +334 if (printStats) { +335 fileStats.finish(); +336 System.out.println("Stats:\n" + fileStats); +337 } +338 +339 reader.close(); +340 } +341 +342 private void scanKeysValues(Path file, KeyValueStatsCollector fileStats, +343 HFileScanner scanner, byte[] row) throws IOException { +344 Cell pCell = null; +345 FileSystem fs = FileSystem.get(getConf()); +346 Set<String> foundMobFiles = new LinkedHashSet<String>(FOUND_MOB_FILES_CACHE_CAPACITY); +347 Set<String> missingMobFiles = new LinkedHashSet<String>(MISSING_MOB_FILES_CACHE_CAPACITY); +348 do { +349 Cell cell = scanner.getCell(); +350 if (row != null && row.length != 0) { +351 int result = CellComparator.COMPARATOR.compareRows(cell, row, 0, row.length); +352 if (result > 0) { +353 break; +354 } else if (result < 0) { +355 continue; +356 } +357 } +358 // collect stats +359 if (printStats) { +360 fileStats.collect(cell); +361 } +362 // dump key value +363 if (printKey) { +364 System.out.print("K: " + cell); +365 if (printValue) { +366 System.out.print(" V: " +367 + Bytes.toStringBinary(cell.getValueArray(), cell.getValueOffset(), +368 cell.getValueLength())); +369 int i = 0; +370 List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), +371 cell.getTagsLength()); +372 for (Tag tag : tags) { +373 System.out.print(String.format(" T[%d]: %s", i++, +374 Bytes.toStringBinary(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength()))); +375 } 376 } -377 } -378 // check if families are consistent -379 if (checkFamily) { -380 String fam = Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), -381 cell.getFamilyLength()); -382 if (!file.toString().contains(fam)) { -383 System.err.println("WARNING, filename does not match kv family," -384 + "\n\tfilename -> " + file + "\n\tkeyvalue -> " +377 System.out.println(); +378 } +379 // check if rows are in order +380 if (checkRow && pCell != null) { +381 if (CellComparator.COMPARATOR.compareRows(pCell, cell) > 0) { +382 System.err.println("WARNING, previous row is greater then" +383 + " current row\n\tfilename -> " + file + "\n\tprevious -> " +384 + CellUtil.getCellKeyAsString(pCell) + "\n\tcurrent -> " 385 + CellUtil.getCellKeyAsString(cell)); 386 } -387 if (pCell != null && CellComparator.compareFamilies(pCell, cell) != 0) { -388 System.err.println("WARNING, previous kv has different family" -389 + " compared to current key\n\tfilename -> " + file -390 + "\n\tprevious -> " + CellUtil.getCellKeyAsString(pCell) -391 + "\n\tcurrent -> " + CellUtil.getCellKeyAsString(cell)); -392 } -393 } -394 // check if mob files are missing. -395 if (checkMobIntegrity && MobUtils.isMobReferenceCell(cell)) { -396 Tag tnTag = MobUtils.getTableNameTag(cell); -397 if (tnTag == null) { -398 System.err.println("ERROR, wrong tag format in mob reference cell " -399 + CellUtil.getCellKeyAsString(cell)); -400 } else if (!MobUtils.hasValidMobRefCellValue(cell)) { -401 System.err.println("ERROR, wrong value format in mob reference cell " -402 + CellUtil.getCellKeyAsString(cell)); -403 } else { -404 TableName tn = TableName.valueOf(tnTag.getValue()); -405 String mobFileName = MobUtils.getMobFileName(cell); -406 boolean exist = mobFileExists(fs, tn, mobFileName, -407 Bytes.toString(CellUtil.cloneFamily(cell)), foundMobFiles, missingMobFiles); -408 if (!exist) { -409 // report error -410 System.err.println("ERROR, the mob file [" + mobFileName -411 + "] is missing referenced by cell " + CellUtil.getCellKeyAsString(cell)); -412 } -413 } -414 } -415 pCell = cell; -416 ++count; -417 } while (scanner.next()); -418 } -419 -420 /** -421 * Checks whether the referenced mob file exists. -422 */ -423 private boolean mobFileExists(FileSystem fs, TableName tn, String mobFileName, String family, -424 Set<String> foundMobFiles, Set<String> missingMobFiles) throws IOException { -425 if (foundMobFiles.contains(mobFileName)) { -426 return true; -427 } -428 if (missingMobFiles.contains(mobFileName)) { -429 return false; -430 } -431 String tableName = tn.getNameAsString(); -432 List<Path> locations = mobFileLocations.get(tableName); -433 if (locations == null) { -434 locations = new ArrayList<Path>(2); -435 locations.add(MobUtils.getMobFamilyPath(getConf(), tn, family)); -436 locations.add(HFileArchiveUtil.getStoreArchivePath(getConf(), tn, -437 MobUtils.getMobRegionInfo(tn).getEncodedName(), family)); -438 mobFileLocations.put(tn.getNameAsString(), locations); -439 } -440 boolean exist = false; -441 for (Path location : locations) { -442 Path mobFilePath = new Path(location, mobFileName); -443 if (fs.exists(mobFilePath)) { -444 exist = true; -445 break; -446 } -447 } -448 if (exist) { -449 evictMobFilesIfNecessary(foundMobFiles, FOUND_MOB_FILES_CACHE_CAPACITY); -450 foundMobFiles.add(mobFileName); -451 } else { -452 evictMobFilesIfNecessary(missingMobFiles, MISSING_MOB_FILES_CACHE_CAPACITY); -453 missingMobFiles.add(mobFileName); -454 } -455 return exist; -456 } -457 -458 /** -459 * Evicts the cached mob files if the set is larger than the limit. -460 */ -461 private void evictMobFilesIfNecessary(Set<String> mobFileNames, int limit) { -462 if (mobFileNames.size() < limit) { -463 return; +387 } +388 // check if families are consistent +389 if (checkFamily) { +390 String fam = Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), +391 cell.getFamilyLength()); +392 if (!file.toString().contains(fam)) { +393 System.err.println("WARNING, filename does not match kv family," +394 + "\n\tfilename -> " + file + "\n\tkeyvalue -> " +395 + CellUtil.getCellKeyAsString(cell)); +396 } +397 if (pCell != null && CellComparator.compareFamilies(pCell, cell) != 0) { +398 System.err.println("WARNING, previous kv has different family" +399 + " compared to current key\n\tfilename -> " + file +400 + "\n\tprevious -> " + CellUtil.getCellKeyAsString(pCell) +401 + "\n\tcurrent -> " + CellUtil.getCellKeyAsString(cell)); +402 } +403 } +404 // check if mob files are missing. +405 if (checkMobIntegrity && MobUtils.isMobReferenceCell(cell)) { +406 Tag tnTag = MobUtils.getTableNameTag(cell); +407 if (tnTag == null) { +408 System.err.println("ERROR, wrong tag format in mob reference cell " +409 + CellUtil.getCellKeyAsString(cell)); +410 } else if (!MobUtils.hasValidMobRefCellValue(cell)) { +411 System.err.println("ERROR, wrong value format in mob reference cell " +412 + CellUtil.getCellKeyAsString(cell)); +413 } else { +414 TableName tn = TableName.valueOf(tnTag.getValue()); +415 String mobFileName = MobUtils.getMobFileName(cell); +416 boolean exist = mobFileExists(fs, tn, mobFileName, +417 Bytes.toString(CellUtil.cloneFamily(cell)), foundMobFiles, missingMobFiles); +418 if (!exist) { +419 // report error +420 System.err.println("ERROR, the mob file [" + mobFileName +421 + "] is missing referenced by cell " + CellUtil.getCellKeyAsString(cell)); +422 } +423 } +424 } +425 pCell = cell; +426 ++count; +427 } while (scanner.next()); +428 } +429 +430 /** +431 * Checks whether the referenced mob file exists. +432 */ +433 private boolean mobFileExists(FileSystem fs, TableName tn, String mobFileName, String family, +434 Set<String> foundMobFiles, Set<String> missingMobFiles) throws IOException { +435 if (foundMobFiles.contains(mobFileName)) { +436 return true; +437 } +438 if (missingMobFiles.contains(mobFileName)) { +439 return false; +440 } +441 String tableName = tn.getNameAsString(); +442 List<Path> locations = mobFileLocations.get(tableName); +443 if (locations == null) { +444 locations = new ArrayList<Path>(2); +445 locations.add(MobUtils.getMobFamilyPath(getConf(), tn, family)); +446 locations.add(HFileArchiveUtil.getStoreArchivePath(getConf(), tn, +447 MobUtils.getMobRegionInfo(tn).getEncodedName(), family)); +448 mobFileLocations.put(tn.getNameAsString(), locations); +449 } +450 boolean exist = false; +451 for (Path location : locations) { +452 Path mobFilePath = new Path(location, mobFileName); +453 if (fs.exists(mobFilePath)) { +454 exist = true; +455 break; +456 } +457 } +458 if (exist) { +459 evictMobFilesIfNecessary(foundMobFiles, FOUND_MOB_FILES_CACHE_CAPACITY); +460 foundMobFiles.add(mobFileName); +461 } else { +462 evictMobFilesIfNecessary(missingMobFiles, MISSING_MOB_FILES_CACHE_CAPACITY); +463 missingMobFiles.add(mobFileName); 464 } -465 int index = 0; -466 int evict = limit / 2; -467 Iterator<String> fileNamesItr = mobFileNames.iterator(); -468 while (index < evict && fileNamesItr.hasNext()) { -469 fileNamesItr.next(); -470 fileNamesItr.remove(); -471 index++; -472 } -473 } -474 -475 /** -476 * Format a string of the form "k1=v1, k2=v2, ..." into separate lines -477 * with a four-space indentation. -478 */ -479 private static String asSeparateLines(String keyValueStr) { -480 return keyValueStr.replaceAll(", ([a-zA-Z]+=)", -481 ",\n" + FOUR_SPACES + "$1"); -482 } -483 -484 private void printMeta(HFile.Reader reader, Map<byte[], byte[]> fileInfo) -485 throws IOException { -486 System.out.println("Block index size as per heapsize: " -487 + reader.indexSize()); -488 System.out.println(asSeparateLines(reader.toString())); -489 System.out.println("Trailer:\n " -490 + asSeparateLines(reader.getTrailer().toString())); -491 System.out.println("Fileinfo:"); -492 for (Map.Entry<byte[], byte[]> e : fileInfo.entrySet()) { -493 System.out.print(FOUR_SPACES + Bytes.toString(e.getKey()) + " = "); -494 if (Bytes.compareTo(e.getKey(), Bytes.toBytes("MAX_SEQ_ID_KEY")) == 0) { -495 long seqid = Bytes.toLong(e.getValue()); -496 System.out.println(seqid); -497 } else if (Bytes.compareTo(e.getKey(), Bytes.toBytes("TIMERANGE")) == 0) { -498 TimeRangeTracker timeRangeTrac