hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1560483 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/regionserver/CompactUtility.java test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
Date Wed, 22 Jan 2014 19:18:31 GMT
Author: liyin
Date: Wed Jan 22 19:18:31 2014
New Revision: 1560483

URL: http://svn.apache.org/r1560483
Log:
[HBASE-10393] Compaction Implemented

Author: abhinavbatra

Summary: Added a cli tool for compacting store files in a region

Test Plan: Unit Test

Reviewers: aaiyer, liyintang, rshroff, adela, manukranthk

Reviewed By: adela

CC: aizatto, hbase-eng@

Differential Revision: https://phabricator.fb.com/D1112260

Task ID: 3058153

Added:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactUtility.java
Modified:
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactUtility.java?rev=1560483&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactUtility.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/CompactUtility.java
Wed Jan 22 19:18:31 2014
@@ -0,0 +1,130 @@
+/**
+ *  Copyright The Apache Software Foundation Licensed to the Apache Software Foundation (ASF)
+ *  under one or more contributor license agreements. See the NOTICE file distributed with
this work
+ *  for additional information regarding copyright ownership. The ASF licenses this file
to you under
+ *  the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance
+ *  with the License. You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed
to in
+ *  writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific
+ *  language governing permissions and limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.commons.cli.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A compaction Utility for the Store files of a region Server.
+ * This utility exposes a hook to compact specific store files.
+ * It provides a command line interface to
+ * compact files without the requirement of running a region server
+ */
+
+public class CompactUtility {
+
+  static final Log LOG = LogFactory.getLog(CompactUtility.class);
+  private String table;
+  private HColumnDescriptor cFamily;
+  private List<Path> filesCompacting;
+  private Configuration conf;
+  private HRegion hRegion;
+
+
+  public CompactUtility(String table, HColumnDescriptor cFamily, long regionId,
+                        List<Path> filesCompacting, Configuration conf)
+          throws IOException {
+    this.conf = conf;
+    this.table = table;
+    this.cFamily = cFamily;
+    this.filesCompacting = filesCompacting;
+    FileSystem fs = FileSystem.get(conf);
+    HTableDescriptor htd = new HTableDescriptor(this.table);
+    htd.addFamily(this.cFamily);
+    HRegionInfo hri = new HRegionInfo(htd, null, null, false, regionId);
+    Path tableDir = HTableDescriptor.getTableDir(
+            new Path(HConstants.HBASE_DIR), this.table.getBytes());
+    Path regionDir = HRegion.getRegionDir(tableDir, hri.getEncodedName());
+    Path rootDir = fs.makeQualified(
+            new Path(conf.get(HConstants.HBASE_DIR)));
+    HLog log = new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
+            new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf, null);
+    this.hRegion = HRegion.openHRegion(hri, rootDir, log, this.conf);
+  }
+
+  public void compact() throws Exception {
+    Store store = hRegion.getStore(this.cFamily.getName());
+    List<StoreFile> storeFiles = new ArrayList<StoreFile>(
+            store.getStorefiles());
+    if (storeFiles == null) {
+      throw new Exception("No Store Files To Compact");
+    }
+    if (filesCompacting != null) {
+      for (Iterator<StoreFile> sFile = storeFiles.iterator(); sFile.hasNext(); ) {
+        if (!filesCompacting.contains(sFile.next().getPath())) {
+          sFile.remove();
+        }
+      }
+    }
+    long maxId = StoreFile.getMaxSequenceIdInList(storeFiles, true);
+    StoreFile.Writer writer = store.compactStores(storeFiles, false, maxId);
+    StoreFile result = store.completeCompaction(storeFiles, writer);
+    store.close();
+    hRegion.close();
+    if (result == null) {
+      throw new Exception("Compaction Failed");
+    }
+  }
+
+  private static void printHelp(Options opt) {
+    new HelpFormatter().printHelp(
+      "Compact Utility < -t tableName -c ColumnFamilyName -r regionID" +
+      " [List of store file Paths]", opt);
+  }
+
+  public static void main(String[] args) throws Exception {
+    Options options = new Options();
+    options.addOption("t", "table name", false,
+            "Table for which the files hold data");
+    options.addOption("c", "Column Family", false,
+            "Column Family of the table");
+    options.addOption("r", "Region Id", false,
+            "Region Id of the region which stores the table");
+    CommandLineParser parser = new PosixParser();
+    CommandLine cmd = parser.parse(options, args);
+    if (!cmd.hasOption("t") || !cmd.hasOption("c") || !cmd.hasOption("r")) {
+      printHelp(options);
+      throw new IOException("Incomplete arguments");
+    }
+    String table = cmd.getOptionValue("t");
+    long regionId = Long.valueOf(cmd.getOptionValue("r"));
+    HColumnDescriptor cFamily = new HColumnDescriptor(cmd.getOptionValue("c"));
+    String[] files = cmd.getArgs();
+    List<Path> filePaths = new ArrayList<Path>();
+    for (String file : files) {
+      filePaths.add(new Path(file));
+    }
+    Configuration conf = HBaseConfiguration.create();
+    SchemaMetrics.configureGlobally(conf);
+    CompactUtility compactUtility = new CompactUtility(
+            table, cFamily, regionId, filePaths, conf);
+    compactUtility.compact();
+  }
+}
+

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1560483&r1=1560482&r2=1560483&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
Wed Jan 22 19:18:31 2014
@@ -26,7 +26,9 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.*;
@@ -34,6 +36,7 @@ import org.apache.hadoop.hbase.io.encodi
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.CompactUtility;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -603,6 +606,48 @@ public class TestCompaction extends HBas
     loader.flushcache();
   }
 
+  /**
+   * Test compaction utility by creating 10 store files
+   * and comparing the number of store files remaining
+   * after compaction
+   *
+   * @throws Exception
+   */
+  public void testCompactionUtility() throws Exception {
+    FileSystem filesystem = FileSystem.get(conf);
+    Path rootdir = filesystem.makeQualified(
+            new Path(conf.get(HConstants.HBASE_DIR)));
+    HTableDescriptor htd = new HTableDescriptor("TestCompaction");
+    htd.addFamily(new HColumnDescriptor("colfamily1"));
+    HRegionInfo hri = new HRegionInfo(htd, null, null);
+    HRegion hRegion = HRegion.createHRegion(hri, rootdir, conf);
+    int nfiles = 10;
+    for (int i = 0; i < nfiles; i++) {
+      createStoreFile(hRegion);
+    }
+    Store store = hRegion.getStore(COLUMN_FAMILY);
+    List<StoreFile> storeFiles = store.getStorefiles();
+    int numFiles1 = storeFiles.size();
+    List<Path> pathList = new ArrayList<Path>();
+    for (StoreFile storeFile : storeFiles) {
+      pathList.add(storeFile.getPath());
+    }
+
+    CompactUtility util = new CompactUtility(
+            "TestCompaction", new HColumnDescriptor("colfamily1"),
+            hri.getRegionId(), pathList, conf);
+    util.compact();
+    Path tableDir = HTableDescriptor.getTableDir(
+            new Path(HConstants.HBASE_DIR), COLUMN_FAMILY);
+    Path regionDir = HRegion.getRegionDir(tableDir, hri.getEncodedName());
+    HLog log = new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
+            new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf, null);
+    hRegion = HRegion.openHRegion(hri, rootdir, log, conf);
+    store = hRegion.getStore(COLUMN_FAMILY);
+    int numfiles2 = store.getNumberOfStoreFiles();
+    assertTrue("CompactionFailed", numfiles2 == 1);
+  }
+
   public void testCompactionWithCorruptResult() throws Exception {
     int nfiles = 10;
     for (int i = 0; i < nfiles; i++) {



Mime
View raw message