hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nspiegelb...@apache.org
Subject svn commit: r1067449 - in /hbase/trunk: CHANGES.txt src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
Date Sat, 05 Feb 2011 14:37:37 GMT
Author: nspiegelberg
Date: Sat Feb  5 14:37:36 2011
New Revision: 1067449

URL: http://svn.apache.org/viewvc?rev=1067449&view=rev
Log:
HBASE-3496 HFile CLI Improvements

Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1067449&r1=1067448&r2=1067449&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Sat Feb  5 14:37:36 2011
@@ -72,6 +72,7 @@ Release 0.91.0 - Unreleased
                copies
    HBASE-3305  Allow round-robin distribution for table created with
                multiple regions (ted yu via jgray)
+   HBASE-3496  HFile CLI Improvements
 
 
   NEW FEATURES

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1067449&r1=1067448&r2=1067449&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Sat Feb  5 14:37:36
2011
@@ -1764,12 +1764,12 @@ public class HFile {
     @Override
     public String toString() {
       StringBuilder sb = new StringBuilder();
-      sb.append("size=" + count);
+      sb.append("size=" + count).append("\n");
       for (int i = 0; i < count ; i++) {
-        sb.append(", ");
-        sb.append("key=").append(Bytes.toStringBinary(blockKeys[i])).
-          append(", offset=").append(blockOffsets[i]).
-          append(", dataSize=" + blockDataSizes[i]);
+        sb.append("key=").append(KeyValue.keyToString(blockKeys[i])).
+          append("\n  offset=").append(blockOffsets[i]).
+          append(", dataSize=" + blockDataSizes[i]).
+          append("\n");
       }
       return sb.toString();
     }
@@ -1890,7 +1890,9 @@ public class HFile {
       Options options = new Options();
       options.addOption("v", "verbose", false, "Verbose output; emits file and meta data
delimiters");
       options.addOption("p", "printkv", false, "Print key/value pairs");
+      options.addOption("e", "printkey", false, "Print keys");
       options.addOption("m", "printmeta", false, "Print meta data of file");
+      options.addOption("b", "printblocks", false, "Print block index meta data");
       options.addOption("k", "checkrow", false,
         "Enable row order check; looks for out-of-order keys");
       options.addOption("a", "checkfamily", false, "Enable family check");
@@ -1906,8 +1908,10 @@ public class HFile {
       CommandLineParser parser = new PosixParser();
       CommandLine cmd = parser.parse(options, args);
       boolean verbose = cmd.hasOption("v");
-      boolean printKeyValue = cmd.hasOption("p");
+      boolean printValue = cmd.hasOption("p");
+      boolean printKey = cmd.hasOption("e") || printValue;
       boolean printMeta = cmd.hasOption("m");
+      boolean printBlocks = cmd.hasOption("b");
       boolean checkRow = cmd.hasOption("k");
       boolean checkFamily = cmd.hasOption("a");
       // get configuration, file system and get list of files
@@ -1916,7 +1920,6 @@ public class HFile {
         conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
       conf.set("fs.default.name",
         conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
-      FileSystem fs = FileSystem.get(conf);
       ArrayList<Path> files = new ArrayList<Path>();
       if (cmd.hasOption("f")) {
         files.add(new Path(cmd.getOptionValue("f")));
@@ -1930,7 +1933,8 @@ public class HFile {
         String enc = HRegionInfo.encodeRegionName(rn);
         Path regionDir = new Path(tableDir, enc);
         if (verbose) System.out.println("region dir -> " + regionDir);
-        List<Path> regionFiles = getStoreFiles(fs, regionDir);
+        List<Path> regionFiles =
+          getStoreFiles(FileSystem.get(conf), regionDir);
         if (verbose) System.out.println("Number of region files found -> " +
           regionFiles.size());
         if (verbose) {
@@ -1944,6 +1948,7 @@ public class HFile {
       // iterate over all files found
       for (Path file : files) {
         if (verbose) System.out.println("Scanning -> " + file);
+        FileSystem fs = file.getFileSystem(conf);
         if (!fs.exists(file)) {
           System.err.println("ERROR, file doesnt exist: " + file);
           continue;
@@ -1952,7 +1957,7 @@ public class HFile {
         HFile.Reader reader = new HFile.Reader(fs, file, null, false, false);
         Map<byte[],byte[]> fileInfo = reader.loadFileInfo();
         int count = 0;
-        if (verbose || printKeyValue || checkRow || checkFamily) {
+        if (verbose || printKey || checkRow || checkFamily) {
           // scan over file and read key/value's and check if requested
           HFileScanner scanner = reader.getScanner(false, false);
           scanner.seekTo();
@@ -1960,9 +1965,12 @@ public class HFile {
           do {
             KeyValue kv = scanner.getKeyValue();
             // dump key value
-            if (printKeyValue) {
-              System.out.println("K: " + kv +
-                  " V: " + Bytes.toStringBinary(kv.getValue()));
+           if (printKey) {
+             System.out.print("K: " + kv);
+             if (printValue) {
+               System.out.print(" V: " + Bytes.toStringBinary(kv.getValue()));
+             }
+             System.out.println();
             }
             // check if rows are in order
             if (checkRow && pkv != null) {
@@ -1992,7 +2000,7 @@ public class HFile {
             count++;
           } while (scanner.next());
         }
-        if (verbose || printKeyValue) {
+        if (verbose || printKey) {
           System.out.println("Scanned kv count -> " + count);
         }
         // print meta data
@@ -2033,6 +2041,10 @@ public class HFile {
             System.out.println("Could not get bloom data from meta block");
           }
         }
+        if (printBlocks) {
+          System.out.println("Block Index:");
+          System.out.println(reader.blockIndex);
+        }
         reader.close();
       }
     } catch (Exception e) {

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java?rev=1067449&r1=1067448&r2=1067449&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java Sat Feb  5
14:37:36 2011
@@ -24,6 +24,7 @@ import java.io.IOException;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
 
 /**
@@ -31,18 +32,28 @@ import org.apache.hadoop.hbase.util.Byte
  */
 public class TestSeekTo extends HBaseTestCase {
 
+  static KeyValue toKV(String row) {
+    return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes
+        .toBytes("qualifier"), Bytes.toBytes("value"));
+  }
+
+  static String toRowStr(KeyValue kv) {
+    return Bytes.toString(kv.getRow());
+  }
+
   Path makeNewFile() throws IOException {
     Path ncTFile = new Path(this.testDir, "basic.hfile");
     FSDataOutputStream fout = this.fs.create(ncTFile);
-    HFile.Writer writer = new HFile.Writer(fout, 40, "none", null);
+    int blocksize = toKV("a").getLength() * 3;
+    HFile.Writer writer = new HFile.Writer(fout, blocksize, "none", null);
     // 4 bytes * 3 * 2 for each key/value +
     // 3 for keys, 15 for values = 42 (woot)
-    writer.append(Bytes.toBytes("c"), Bytes.toBytes("value"));
-    writer.append(Bytes.toBytes("e"), Bytes.toBytes("value"));
-    writer.append(Bytes.toBytes("g"), Bytes.toBytes("value"));
+    writer.append(toKV("c"));
+    writer.append(toKV("e"));
+    writer.append(toKV("g"));
     // block transition
-    writer.append(Bytes.toBytes("i"), Bytes.toBytes("value"));
-    writer.append(Bytes.toBytes("k"), Bytes.toBytes("value"));
+    writer.append(toKV("i"));
+    writer.append(toKV("k"));
     writer.close();
     fout.close();
     return ncTFile;
@@ -52,32 +63,32 @@ public class TestSeekTo extends HBaseTes
     HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
     reader.loadFileInfo();
     HFileScanner scanner = reader.getScanner(false, true);
-    assertEquals(false, scanner.seekBefore(Bytes.toBytes("a")));
+    assertEquals(false, scanner.seekBefore(toKV("a").getKey()));
 
-    assertEquals(false, scanner.seekBefore(Bytes.toBytes("c")));
+    assertEquals(false, scanner.seekBefore(toKV("c").getKey()));
 
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("d")));
-    assertEquals("c", scanner.getKeyString());
+    assertEquals(true, scanner.seekBefore(toKV("d").getKey()));
+    assertEquals("c", toRowStr(scanner.getKeyValue()));
 
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("e")));
-    assertEquals("c", scanner.getKeyString());
-
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("f")));
-    assertEquals("e", scanner.getKeyString());
-
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("g")));
-    assertEquals("e", scanner.getKeyString());
-
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("h")));
-    assertEquals("g", scanner.getKeyString());
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("i")));
-    assertEquals("g", scanner.getKeyString());
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("j")));
-    assertEquals("i", scanner.getKeyString());
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("k")));
-    assertEquals("i", scanner.getKeyString());
-    assertEquals(true, scanner.seekBefore(Bytes.toBytes("l")));
-    assertEquals("k", scanner.getKeyString());
+    assertEquals(true, scanner.seekBefore(toKV("e").getKey()));
+    assertEquals("c", toRowStr(scanner.getKeyValue()));
+
+    assertEquals(true, scanner.seekBefore(toKV("f").getKey()));
+    assertEquals("e", toRowStr(scanner.getKeyValue()));
+
+    assertEquals(true, scanner.seekBefore(toKV("g").getKey()));
+    assertEquals("e", toRowStr(scanner.getKeyValue()));
+
+    assertEquals(true, scanner.seekBefore(toKV("h").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+    assertEquals(true, scanner.seekBefore(toKV("i").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
+    assertEquals(true, scanner.seekBefore(toKV("j").getKey()));
+    assertEquals("i", toRowStr(scanner.getKeyValue()));
+    assertEquals(true, scanner.seekBefore(toKV("k").getKey()));
+    assertEquals("i", toRowStr(scanner.getKeyValue()));
+    assertEquals(true, scanner.seekBefore(toKV("l").getKey()));
+    assertEquals("k", toRowStr(scanner.getKeyValue()));
   }
 
   public void testSeekTo() throws Exception {
@@ -87,17 +98,17 @@ public class TestSeekTo extends HBaseTes
     assertEquals(2, reader.blockIndex.count);
     HFileScanner scanner = reader.getScanner(false, true);
     // lies before the start of the file.
-    assertEquals(-1, scanner.seekTo(Bytes.toBytes("a")));
+    assertEquals(-1, scanner.seekTo(toKV("a").getKey()));
 
-    assertEquals(1, scanner.seekTo(Bytes.toBytes("d")));
-    assertEquals("c", scanner.getKeyString());
+    assertEquals(1, scanner.seekTo(toKV("d").getKey()));
+    assertEquals("c", toRowStr(scanner.getKeyValue()));
 
     // Across a block boundary now.
-    assertEquals(1, scanner.seekTo(Bytes.toBytes("h")));
-    assertEquals("g", scanner.getKeyString());
+    assertEquals(1, scanner.seekTo(toKV("h").getKey()));
+    assertEquals("g", toRowStr(scanner.getKeyValue()));
 
-    assertEquals(1, scanner.seekTo(Bytes.toBytes("l")));
-    assertEquals("k", scanner.getKeyString());
+    assertEquals(1, scanner.seekTo(toKV("l").getKey()));
+    assertEquals("k", toRowStr(scanner.getKeyValue()));
   }
 
   public void testBlockContainingKey() throws Exception {
@@ -105,19 +116,27 @@ public class TestSeekTo extends HBaseTes
     HFile.Reader reader = new HFile.Reader(fs, p, null, false, false);
     reader.loadFileInfo();
     System.out.println(reader.blockIndex.toString());
+    int klen = toKV("a").getKey().length;
     // falls before the start of the file.
-    assertEquals(-1, reader.blockIndex.blockContainingKey(Bytes.toBytes("a"), 0, 1));
-    assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("c"), 0, 1));
-    assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("d"), 0, 1));
-    assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("e"), 0, 1));
-    assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("g"), 0, 1));
-    assertEquals(0, reader.blockIndex.blockContainingKey(Bytes.toBytes("h"), 0, 1));
-    assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("i"), 0, 1));
-    assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("j"), 0, 1));
-    assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("k"), 0, 1));
-    assertEquals(1, reader.blockIndex.blockContainingKey(Bytes.toBytes("l"), 0, 1));
-
-
-
+    assertEquals(-1, reader.blockIndex.blockContainingKey(toKV("a").getKey(),
+        0, klen));
+    assertEquals(0, reader.blockIndex.blockContainingKey(toKV("c").getKey(), 0,
+        klen));
+    assertEquals(0, reader.blockIndex.blockContainingKey(toKV("d").getKey(), 0,
+        klen));
+    assertEquals(0, reader.blockIndex.blockContainingKey(toKV("e").getKey(), 0,
+        klen));
+    assertEquals(0, reader.blockIndex.blockContainingKey(toKV("g").getKey(), 0,
+        klen));
+    assertEquals(0, reader.blockIndex.blockContainingKey(toKV("h").getKey(), 0,
+        klen));
+    assertEquals(1, reader.blockIndex.blockContainingKey(toKV("i").getKey(), 0,
+        klen));
+    assertEquals(1, reader.blockIndex.blockContainingKey(toKV("j").getKey(), 0,
+        klen));
+    assertEquals(1, reader.blockIndex.blockContainingKey(toKV("k").getKey(), 0,
+        klen));
+    assertEquals(1, reader.blockIndex.blockContainingKey(toKV("l").getKey(), 0,
+        klen));
   }
-}
\ No newline at end of file
+}



Mime
View raw message