hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zg...@apache.org
Subject hbase git commit: HBASE-17596 Implement add/delete/modify column family methods
Date Mon, 06 Feb 2017 05:47:10 GMT
Repository: hbase
Updated Branches:
  refs/heads/master d22bfc036 -> 5093a49e0


HBASE-17596 Implement add/delete/modify column family methods


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5093a49e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5093a49e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5093a49e

Branch: refs/heads/master
Commit: 5093a49e0ba3e8f88e03ed8b81dd3abada402c3a
Parents: d22bfc0
Author: Guanghao Zhang <zghao@apache.org>
Authored: Mon Feb 6 13:45:01 2017 +0800
Committer: Guanghao Zhang <zghao@apache.org>
Committed: Mon Feb 6 13:45:01 2017 +0800

----------------------------------------------------------------------
 .../apache/hadoop/hbase/client/AsyncAdmin.java  |  35 ++++
 .../hadoop/hbase/client/AsyncHBaseAdmin.java    |  81 ++++++++
 .../hadoop/hbase/client/TestAsyncAdmin.java     | 192 +++++++++++++++++++
 3 files changed, 308 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/5093a49e/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index 29b98de..5c6a390 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -20,10 +20,12 @@ package org.apache.hadoop.hbase.client;
 import java.util.concurrent.CompletableFuture;
 import java.util.regex.Pattern;
 
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Pair;
 
 /**
  *  The asynchronous administrative API for HBase.
@@ -217,6 +219,39 @@ public interface AsyncAdmin {
   CompletableFuture<HTableDescriptor[]> disableTables(Pattern pattern);
 
   /**
+   * Get the status of alter command - indicates how many regions have received the updated
schema
+   * Asynchronous operation.
+   * @param tableName TableName instance
+   * @return Pair indicating the number of regions updated Pair.getFirst() is the regions
that are
+   *         yet to be updated Pair.getSecond() is the total number of regions of the table.
The
+   *         return value will be wrapped by a {@link CompletableFuture}.
+   */
+  CompletableFuture<Pair<Integer, Integer>> getAlterStatus(final TableName tableName);
+
+  /**
+   * Add a column family to an existing table.
+   * @param tableName name of the table to add column family to
+   * @param columnFamily column family descriptor of column family to be added
+   */
+  CompletableFuture<Void> addColumnFamily(final TableName tableName,
+      final HColumnDescriptor columnFamily);
+
+  /**
+   * Delete a column family from a table.
+   * @param tableName name of table
+   * @param columnFamily name of column family to be deleted
+   */
+  CompletableFuture<Void> deleteColumnFamily(final TableName tableName, final byte[]
columnFamily);
+
+  /**
+   * Modify an existing column family on a table.
+   * @param tableName name of table
+   * @param columnFamily new column family descriptor to use
+   */
+  CompletableFuture<Void> modifyColumnFamily(final TableName tableName,
+      final HColumnDescriptor columnFamily);
+
+  /**
    * Turn the load balancer on or off.
    * @param on
    * @return Previous balancer value wrapped by a {@link CompletableFuture}.

http://git-wip-us.apache.org/repos/asf/hbase/blob/5093a49e/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
index fecdb4f..89cd276 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -30,6 +30,7 @@ import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.AsyncMetaTableAccessor;
@@ -43,14 +44,20 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback;
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TableSchema;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AddColumnRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AddColumnResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DisableTableRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DisableTableResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.EnableTableRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.EnableTableResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DeleteColumnRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DeleteColumnResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetProcedureResultRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetProcedureResultResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableNamesRequest;
@@ -62,12 +69,15 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DeleteTabl
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsBalancerEnabledRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsBalancerEnabledResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
+import org.apache.hadoop.hbase.util.Pair;
 
 /**
  * The implementation of AsyncAdmin.
@@ -369,6 +379,44 @@ public class AsyncHBaseAdmin implements AsyncAdmin {
     return batchTableOperations(pattern, (table) -> disableTable(table), "DISABLE");
   }
 
+
+  @Override
+  public CompletableFuture<Pair<Integer, Integer>> getAlterStatus(TableName tableName)
{
+    return this
+        .<Pair<Integer, Integer>> newCaller()
+        .action(
+          (controller, stub) -> this
+              .<GetSchemaAlterStatusRequest, GetSchemaAlterStatusResponse, Pair<Integer,
Integer>> call(
+                controller, stub, RequestConverter.buildGetSchemaAlterStatusRequest(tableName),
(s,
+                    c, req, done) -> s.getSchemaAlterStatus(c, req, done), (resp) ->
new Pair<>(
+                    resp.getYetToUpdateRegions(), resp.getTotalRegions()))).call();
+  }
+
+  @Override
+  public CompletableFuture<Void> addColumnFamily(TableName tableName, HColumnDescriptor
columnFamily) {
+    return this.<AddColumnRequest, AddColumnResponse> procedureCall(
+      RequestConverter.buildAddColumnRequest(tableName, columnFamily, ng.getNonceGroup(),
+        ng.newNonce()), (s, c, req, done) -> s.addColumn(c, req, done), (resp) -> resp.getProcId(),
+      new AddColumnFamilyProcedureBiConsumer(this, tableName));
+  }
+
+  @Override
+  public CompletableFuture<Void> deleteColumnFamily(TableName tableName, byte[] columnFamily)
{
+    return this.<DeleteColumnRequest, DeleteColumnResponse> procedureCall(
+      RequestConverter.buildDeleteColumnRequest(tableName, columnFamily, ng.getNonceGroup(),
+        ng.newNonce()), (s, c, req, done) -> s.deleteColumn(c, req, done),
+      (resp) -> resp.getProcId(), new DeleteColumnFamilyProcedureBiConsumer(this, tableName));
+  }
+
+  @Override
+  public CompletableFuture<Void> modifyColumnFamily(TableName tableName,
+      HColumnDescriptor columnFamily) {
+    return this.<ModifyColumnRequest, ModifyColumnResponse> procedureCall(
+      RequestConverter.buildModifyColumnRequest(tableName, columnFamily, ng.getNonceGroup(),
+        ng.newNonce()), (s, c, req, done) -> s.modifyColumn(c, req, done),
+      (resp) -> resp.getProcId(), new ModifyColumnFamilyProcedureBiConsumer(this, tableName));
+  }
+
   @Override
   public CompletableFuture<Boolean> setBalancerRunning(final boolean on) {
     return this
@@ -531,6 +579,39 @@ public class AsyncHBaseAdmin implements AsyncAdmin {
     }
   }
 
+  private class AddColumnFamilyProcedureBiConsumer extends TableProcedureBiConsumer {
+
+    AddColumnFamilyProcedureBiConsumer(AsyncAdmin admin, TableName tableName) {
+      super(admin, tableName);
+    }
+
+    String getOperationType() {
+      return "ADD_COLUMN_FAMILY";
+    }
+  }
+
+  private class DeleteColumnFamilyProcedureBiConsumer extends TableProcedureBiConsumer {
+
+    DeleteColumnFamilyProcedureBiConsumer(AsyncAdmin admin, TableName tableName) {
+      super(admin, tableName);
+    }
+
+    String getOperationType() {
+      return "DELETE_COLUMN_FAMILY";
+    }
+  }
+
+  private class ModifyColumnFamilyProcedureBiConsumer extends TableProcedureBiConsumer {
+
+    ModifyColumnFamilyProcedureBiConsumer(AsyncAdmin admin, TableName tableName) {
+      super(admin, tableName);
+    }
+
+    String getOperationType() {
+      return "MODIFY_COLUMN_FAMILY";
+    }
+  }
+
   private CompletableFuture<Void> waitProcedureResult(CompletableFuture<Long>
procFuture) {
     CompletableFuture<Void> future = new CompletableFuture<>();
     procFuture.whenComplete((procId, error) -> {

http://git-wip-us.apache.org/repos/asf/hbase/blob/5093a49e/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdmin.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdmin.java
index 8ba5dc7..7f6c2ad 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdmin.java
@@ -32,12 +32,14 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.CompletionException;
 import java.util.regex.Pattern;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.AsyncMetaTableAccessor;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -45,15 +47,20 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.TableNotEnabledException;
 import org.apache.hadoop.hbase.constraint.ConstraintException;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -68,6 +75,8 @@ public class TestAsyncAdmin {
   private static final Log LOG = LogFactory.getLog(TestAdmin1.class);
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static byte[] FAMILY = Bytes.toBytes("testFamily");
+  private static final byte[] FAMILY_0 = Bytes.toBytes("cf0");
+  private static final byte[] FAMILY_1 = Bytes.toBytes("cf1");
 
   private static AsyncConnection ASYNC_CONN;
   private AsyncAdmin admin;
@@ -683,6 +692,189 @@ public class TestAsyncAdmin {
     admin.createTable(htd).join();
   }
 
+  @Test
+  public void testAddColumnFamily() throws IOException {
+    TableName TABLE_NAME = TableName.valueOf("testAddColumnFamily");
+    // Create a table with two families
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+    admin.createTable(baseHtd).join();
+    admin.disableTable(TABLE_NAME).join();
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+
+      // Modify the table removing one family and verify the descriptor
+      admin.addColumnFamily(TABLE_NAME, new HColumnDescriptor(FAMILY_1)).join();
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1);
+    } finally {
+      admin.deleteTable(TABLE_NAME);
+    }
+  }
+
+  @Test
+  public void testAddSameColumnFamilyTwice() throws Exception {
+    TableName TABLE_NAME = TableName.valueOf("testAddSameColumnFamilyTwice");
+    // Create a table with one families
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+    admin.createTable(baseHtd).join();
+    admin.disableTable(TABLE_NAME).join();
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+
+      // Modify the table removing one family and verify the descriptor
+      this.admin.addColumnFamily(TABLE_NAME, new HColumnDescriptor(FAMILY_1)).join();
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1);
+
+      try {
+        // Add same column family again - expect failure
+        this.admin.addColumnFamily(TABLE_NAME, new HColumnDescriptor(FAMILY_1)).join();
+        Assert.fail("Delete a non-exist column family should fail");
+      } catch (Exception e) {
+        // Expected.
+      }
+    } finally {
+      admin.deleteTable(TABLE_NAME).join();
+    }
+  }
+
+  @Test
+  public void testModifyColumnFamily() throws Exception {
+    TableName TABLE_NAME = TableName.valueOf("testModifyColumnFamily");
+
+    HColumnDescriptor cfDescriptor = new HColumnDescriptor(FAMILY_0);
+    int blockSize = cfDescriptor.getBlocksize();
+    // Create a table with one families
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(cfDescriptor);
+    admin.createTable(baseHtd).join();
+    admin.disableTable(TABLE_NAME).join();
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+
+      int newBlockSize = 2 * blockSize;
+      cfDescriptor.setBlocksize(newBlockSize);
+
+      // Modify colymn family
+      admin.modifyColumnFamily(TABLE_NAME, cfDescriptor).join();
+
+      HTableDescriptor htd = admin.getTableDescriptor(TABLE_NAME).get();
+      HColumnDescriptor hcfd = htd.getFamily(FAMILY_0);
+      assertTrue(hcfd.getBlocksize() == newBlockSize);
+    } finally {
+      admin.deleteTable(TABLE_NAME).join();
+    }
+  }
+
+  @Test
+  public void testModifyNonExistingColumnFamily() throws IOException {
+    TableName TABLE_NAME = TableName.valueOf("testModifyNonExistingColumnFamily");
+
+    HColumnDescriptor cfDescriptor = new HColumnDescriptor(FAMILY_1);
+    int blockSize = cfDescriptor.getBlocksize();
+    // Create a table with one families
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+    admin.createTable(baseHtd).join();
+    admin.disableTable(TABLE_NAME).join();
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+
+      int newBlockSize = 2 * blockSize;
+      cfDescriptor.setBlocksize(newBlockSize);
+
+      // Modify a column family that is not in the table.
+      try {
+        admin.modifyColumnFamily(TABLE_NAME, cfDescriptor).join();
+        Assert.fail("Modify a non-exist column family should fail");
+      } catch (Exception e) {
+        // Expected.
+      }
+    } finally {
+      admin.deleteTable(TABLE_NAME).join();
+    }
+  }
+
+  @Test
+  public void testDeleteColumnFamily() throws IOException {
+    TableName TABLE_NAME = TableName.valueOf("testDeleteColumnFamily");
+    // Create a table with two families
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_1));
+    admin.createTable(baseHtd).join();
+    admin.disableTable(TABLE_NAME).join();
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1);
+
+      // Modify the table removing one family and verify the descriptor
+      admin.deleteColumnFamily(TABLE_NAME, FAMILY_1).join();
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+    } finally {
+      admin.deleteTable(TABLE_NAME).join();
+    }
+  }
+
+  @Test
+  public void testDeleteSameColumnFamilyTwice() throws IOException {
+    TableName TABLE_NAME = TableName.valueOf("testDeleteSameColumnFamilyTwice");
+    // Create a table with two families
+    HTableDescriptor baseHtd = new HTableDescriptor(TABLE_NAME);
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_0));
+    baseHtd.addFamily(new HColumnDescriptor(FAMILY_1));
+    admin.createTable(baseHtd).join();
+    admin.disableTable(TABLE_NAME).join();
+    try {
+      // Verify the table descriptor
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1);
+
+      // Modify the table removing one family and verify the descriptor
+      admin.deleteColumnFamily(TABLE_NAME, FAMILY_1).join();
+      verifyTableDescriptor(TABLE_NAME, FAMILY_0);
+
+      try {
+        // Delete again - expect failure
+        admin.deleteColumnFamily(TABLE_NAME, FAMILY_1).join();
+        Assert.fail("Delete a non-exist column family should fail");
+      } catch (Exception e) {
+        // Expected.
+      }
+    } finally {
+      admin.deleteTable(TABLE_NAME).join();
+    }
+  }
+
+  private void verifyTableDescriptor(final TableName tableName, final byte[]... families)
+      throws IOException {
+    Admin admin = TEST_UTIL.getAdmin();
+
+    // Verify descriptor from master
+    HTableDescriptor htd = admin.getTableDescriptor(tableName);
+    verifyTableDescriptor(htd, tableName, families);
+
+    // Verify descriptor from HDFS
+    MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+    Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
+    HTableDescriptor td = FSTableDescriptors
+        .getTableDescriptorFromFs(mfs.getFileSystem(), tableDir);
+    verifyTableDescriptor(td, tableName, families);
+  }
+
+  private void verifyTableDescriptor(final HTableDescriptor htd, final TableName tableName,
+      final byte[]... families) {
+    Set<byte[]> htdFamilies = htd.getFamiliesKeys();
+    assertEquals(tableName, htd.getTableName());
+    assertEquals(families.length, htdFamilies.size());
+    for (byte[] familyName : families) {
+      assertTrue("Expected family " + Bytes.toString(familyName), htdFamilies.contains(familyName));
+    }
+  }
+
   @Test(timeout = 30000)
   public void testBalancer() throws Exception {
     boolean initialState = admin.isBalancerEnabled().get();


Mime
View raw message