hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1584180 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/client/ main/java/org/apache/hadoop/hbase/ipc/ main/java/org/apache/hadoop/hbase/master/ test/java/org/apache/hadoop/hbase/ test...
Date Wed, 02 Apr 2014 21:03:35 GMT
Author: liyin
Date: Wed Apr  2 21:03:34 2014
New Revision: 1584180

URL: http://svn.apache.org/r1584180
Log:
[HBASE-10865] Store per table server assignment preference

Author: elliott

Summary:
* Allow user to add server perference in HTableDescriptor
* Honor that preference when creating the table.
* Add tests for ser/deser of server lists in HTD.

Test Plan:
* Wrote unit tests for HTD ser/deser
* Changed previous test that creates table on servers

iloveunittests

Reviewers: adela

Reviewed By: adela

CC: hbase-eng@

Differential Revision: https://phabricator.fb.com/D1246832

Task ID: 4023938

Added:
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java
      - copied, changed from r1584179, hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java
Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java?rev=1584180&r1=1584179&r2=1584180&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java Wed
Apr  2 21:03:34 2014
@@ -35,6 +35,9 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -50,6 +53,7 @@ import com.facebook.swift.codec.ThriftSt
  */
 @ThriftStruct
 public class HTableDescriptor implements WritableComparable<HTableDescriptor> {
+  public static final Log LOG = LogFactory.getLog(HTableDescriptor.class);
 
   // Changes prior to version 3 were not recorded here.
   // Version 3 adds metadata as a map where keys and values are byte[].
@@ -92,6 +96,10 @@ public class HTableDescriptor implements
   public static final ImmutableBytesWritable DISABLE_WAL_KEY =
     new ImmutableBytesWritable(Bytes.toBytes(DISABLE_WAL));
 
+  public static final String SERVER_SET = "SERVER_SET";
+  public static final ImmutableBytesWritable SERVER_SET_KEY =
+      new ImmutableBytesWritable( Bytes.toBytes(SERVER_SET));
+
 
   // The below are ugly but better than creating them each time till we
   // replace booleans being saved as Strings with plain booleans.  Need a
@@ -138,6 +146,9 @@ public class HTableDescriptor implements
   public final Map<byte [], HColumnDescriptor> families =
     new TreeMap<byte [], HColumnDescriptor>(Bytes.BYTES_RAWCOMPARATOR);
 
+  // Used to store what servers this can be assigned to.
+  private ServerSet serverSet = null;
+
   /**
    * Private constructor used internally creating table descriptors for
    * catalog tables: e.g. .META. and -ROOT-.
@@ -821,11 +832,75 @@ public class HTableDescriptor implements
   }
 
   /**
-   * @param readOnly True if all of the columns in the table should be read
+   * @param disable should the wal be disabled for this table.
    * only.
    */
   public void setWALDisabled(final boolean disable) {
     setValue(DISABLE_WAL_KEY, disable? TRUE: FALSE);
   }
 
+  /**
+   * This is the class serialized into SERVER_SET_KEY to hold the list of servers.
+   */
+  @ThriftStruct
+  public static class ServerSet {
+
+    private Set<HServerAddress> servers = null;
+
+    @ThriftConstructor
+    public ServerSet(Set<HServerAddress> servers) {
+      this.servers = servers;
+    }
+
+    @ThriftField(1)
+    public Set<HServerAddress> getServers() {
+      return servers;
+    }
+
+    public void setServers(Set<HServerAddress> servers) {
+      this.servers = servers;
+    }
+  }
+
+  public synchronized void setServers(Collection<HServerAddress> servers) {
+    if (this.serverSet == null) {
+      this.serverSet = new ServerSet(null);
+    }
+
+    if ( servers == null) {
+      this.serverSet.servers = null;
+      return;
+    }
+
+    if (servers.size() < 3 ) {
+      throw new IllegalArgumentException("Must provide at least three servers.");
+    }
+
+    this.serverSet.setServers(ImmutableSet.copyOf(servers));
+    try {
+      // Hex encode everything so that the HTD can still be printed everywhere.
+      String encodedStringSet = Bytes.writeThriftBytesAndGetString(serverSet, ServerSet.class);
+      this.setValue(SERVER_SET_KEY, Bytes.toBytes(encodedStringSet));
+    } catch (Exception e) {
+      LOG.error("Error serializing server set from HTableDescriptor", e);
+    }
+  }
+
+  public synchronized Set<HServerAddress> getServers() {
+    if (serverSet == null) {
+      byte[] serverSetBytes = getValue(SERVER_SET_KEY);
+      if (serverSetBytes != null) {
+        try {
+          byte[] decodedBytes = Bytes.hexToBytes(Bytes.toString(serverSetBytes));
+          serverSet = Bytes.readThriftBytes(decodedBytes, ServerSet.class);
+        } catch (Exception e) {
+          LOG.error("Error de-serializing server set into HTableDescriptor", e);
+        }
+      }
+      if (serverSet == null) {
+        serverSet = new ServerSet(null);
+      }
+    }
+    return this.serverSet.getServers();
+  }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1584180&r1=1584179&r2=1584180&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Wed
Apr  2 21:03:34 2014
@@ -270,20 +270,6 @@ public class HBaseAdmin {
     checkTableOnline(desc, splitKeys);
   }
 
-  /**
-   * Same like {@link #createTable(HTableDescriptor, byte[][])} but creates
-   * table on specific regionservers
-   *
-   * @throws IOException
-   */
-  public void createTable(final HTableDescriptor desc, byte[][] splitKeys,
-      List<HServerAddress> servers) throws IOException {
-    HTableDescriptor.isLegalTableName(desc.getName());
-    checkSplitKeys(splitKeys);
-    createTableAsyncAndPlaceOnServers(desc, splitKeys, servers);
-    checkTableOnline(desc, splitKeys);
-  }
-
   private void checkTableOnline(final HTableDescriptor desc, byte[][] splitKeys)
       throws IOException, RegionOfflineException, InterruptedIOException {
     int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
@@ -383,30 +369,6 @@ public class HBaseAdmin {
     }
   }
 
-  /**
-   * SAme as {@link #createTableAsync(HTableDescriptor, byte[][])} but using a
-   * specific set of servers to assign the regions
-   *
-   * @param desc
-   * @param splitKeys
-   * @param servers
-   * @throws IOException
-   */
-  public void createTableAsyncAndPlaceOnServers(HTableDescriptor desc,
-      byte[][] splitKeys, List<HServerAddress> servers) throws IOException {
-    if (this.master == null) {
-      throw new MasterNotRunningException("master has been shut down");
-    }
-    HTableDescriptor.isLegalTableName(desc.getName());
-    try {
-      this.master.createTableAndAssignOnServers(desc, splitKeys, servers);
-    } catch (RemoteException e) {
-      throw RemoteExceptionHandler.decodeRemoteException(e);
-    } catch (SocketTimeoutException ste) {
-      LOG.warn("Creating " + desc.getNameAsString() + " took too long", ste);
-    }
-  }
-
 
   /**
    * Deletes a table.

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java?rev=1584180&r1=1584179&r2=1584180&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
Wed Apr  2 21:03:34 2014
@@ -59,17 +59,6 @@ public interface HMasterInterface extend
   throws IOException;
 
   /**
-   * Same like {@link #createTable(HTableDescriptor, byte[][])} above, but we
-   * specify the set of machines where we want the regions to be assigned
-   *
-   * @param desc
-   * @param splitKeys
-   * @throws IOException
-   */
-  public void createTableAndAssignOnServers(HTableDescriptor desc,
-      byte[][] splitKeys, List<HServerAddress> servers) throws IOException;
-
-  /**
    * Deletes a table
    * @param tableName table to delete
    * @throws IOException e

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1584180&r1=1584179&r2=1584180&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Wed Apr
 2 21:03:34 2014
@@ -33,6 +33,7 @@ import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -1484,31 +1485,6 @@ public class HMaster extends HasThread i
     }
   }
 
-  @Override
-  public void createTableAndAssignOnServers(HTableDescriptor desc,
-      byte[][] splitKeys, List<HServerAddress> servers) throws IOException {
-    HRegionInfo[] newRegions = createRegionsForNewTable(desc, splitKeys);
-    try {
-      // We can not create a table unless meta regions have already been
-      // assigned and scanned.
-      if (!this.regionManager.areAllMetaRegionsOnline()) {
-        throw new NotAllMetaRegionsOnlineException();
-      }
-      if (!this.serverManager.hasEnoughRegionServers()) {
-        throw new IOException("not enough servers to create table yet");
-      }
-      createTable(newRegions, servers);
-      LOG.info("Succeeded in creating table " + desc.getNameAsString()
-          + "on the specified servers");
-    } catch (TableExistsException e) {
-      throw e;
-    } catch (IOException e) {
-      LOG.error("Cannot create table " + desc.getNameAsString()
-          + " because of " + e.toString() + " on the specified servers");
-      throw RemoteExceptionHandler.checkIOException(e);
-    }
-  }
-
   private static boolean tableExists(HRegionInterface srvr,
       byte[] metaRegionName, String tableName)
       throws IOException {
@@ -1534,40 +1510,23 @@ public class HMaster extends HasThread i
     return false;
   }
 
-  private synchronized void createTable(final HRegionInfo[] newRegions)
-      throws IOException {
-    String tableName = newRegions[0].getTableDesc().getNameAsString();
-    AssignmentPlan assignmentPlan = null;
-    if (this.shouldAssignRegionsWithFavoredNodes) {
-      // Get the assignment domain for this table
-      AssignmentDomain domain = this.getAssignmentDomain(tableName);
-      // Get the assignment plan for the new regions
-      assignmentPlan = regionPlacement.getNewAssignmentPlan(newRegions, domain);
-    }
-    createTable(newRegions, assignmentPlan);
-  }
-
   /**
-   * Create table such that we place the new regions on the specified machines
+   * Create table.  If the HTableDescriptor has servers listed the regions will be placed
+   * on those servers.
+   *
    * @param newRegions - new regions from the new table
-   * @param servers - set of machines where we like the regions to be placed
    * @throws IOException
    */
-  private synchronized void createTable(final HRegionInfo[] newRegions,
-      List<HServerAddress> servers) throws IOException {
+  private synchronized void createTable(final HRegionInfo[] newRegions) throws IOException
{
     String tableName = newRegions[0].getTableDesc().getNameAsString();
     AssignmentPlan assignmentPlan = null;
     if (this.shouldAssignRegionsWithFavoredNodes) {
       // Get the assignment domain for this table
-      AssignmentDomain domain = this.getAssignmentDomain(tableName, servers);
+      AssignmentDomain domain = this.getAssignmentDomain(newRegions[0].getTableDesc());
       // Get the assignment plan for the new regions
       assignmentPlan = regionPlacement.getNewAssignmentPlan(newRegions, domain);
     }
-    createTable(newRegions, assignmentPlan);
-  }
 
-  private synchronized void createTable(final HRegionInfo[] newRegions, AssignmentPlan assignmentPlan)
throws IOException {
-    String tableName = newRegions[0].getTableDesc().getNameAsString();
     // 1. Check to see if table already exists. Get meta region where
     // table would sit should it exist. Open scanner on it. If a region
     // for the table we want to create already exists, then table already
@@ -1617,30 +1576,23 @@ public class HMaster extends HasThread i
    * Currently the domain would be generated by shuffling all the online
    * region servers.
    *
-   * It would be easy to extend for the multi-tenancy in the future.
-   * @param tableName
+   * @param htd
    * @return the assignment domain for the table.
    */
-  private AssignmentDomain getAssignmentDomain(String tableName) {
+  private AssignmentDomain getAssignmentDomain(HTableDescriptor htd) {
     // Get all the online region servers
-    List<HServerAddress> onlineRSList =
-      this.serverManager.getOnlineRegionServerList();
-    return getAssignmentDomain(tableName, onlineRSList);
-  }
+    List<HServerAddress> servers = null;
+
+    if (htd.getServers() == null) {
+      servers = this.serverManager.getOnlineRegionServerList();
+    } else {
+      Set<HServerAddress> hServerAddresses = new HashSet<>(htd.getServers());
+      hServerAddresses.retainAll(this.serverManager.getOnlineRegionServerList());
+      servers = Lists.newArrayList(hServerAddresses);
+    }
 
-  /**
-   * Get the assignment domain for the table. Currently the domain would be
-   * generated by shuffling the passed region servers.
-   *
-   * It would be easy to extend for the multi-tenancy in the future.
-   *
-   * @param tableName
-   * @param servers - list of servers that we want to be included in the plan
-   * @return the assignment domain for the table.
-   */
-  private AssignmentDomain getAssignmentDomain(String tableName, List<HServerAddress>
servers) {
     // Shuffle the server list based on the tableName
-    Random random = new Random(tableName.hashCode());
+    Random random = new Random(htd.getNameAsString().hashCode());
     Collections.shuffle(servers, random);
     // Add the shuffled server list into the assignment domain
     AssignmentDomain domain = new AssignmentDomain(this.conf);

Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java?rev=1584180&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
(added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestHTableDescriptor.java
Wed Apr  2 21:03:34 2014
@@ -0,0 +1,120 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+public class TestHTableDescriptor {
+  @Test
+  public void testGetNameAsString() throws Exception {
+    HTableDescriptor htd = new HTableDescriptor(Bytes.toBytes("Test"));
+
+    // Make sure the to and from string works
+    assertEquals("Test", htd.getNameAsString());
+
+    // Make sure the string instance is cached.
+    assertSame(htd.getNameAsString(), htd.getNameAsString());
+  }
+
+  @Test
+  public void testWriteAndReadFieldsWithServerSet() throws Exception {
+    HTableDescriptor htd = new HTableDescriptor("TestTable");
+    htd.addFamily(new HColumnDescriptor("d"));
+    htd.setMaxFileSize(Long.MAX_VALUE - 1);
+    htd.setWALDisabled(true);
+
+
+    Set<HServerAddress> servers = new HashSet<>();
+
+    servers.add(new HServerAddress(java.net.InetAddress.getLocalHost().getHostName(),0));
+    htd.setServers(servers);
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream out = new DataOutputStream(baos);
+    htd.write(out);
+
+    byte[] buf = baos.toByteArray();
+
+    DataInputBuffer in = new DataInputBuffer();
+    in.reset(buf, 0, buf.length);
+
+    HTableDescriptor n = new HTableDescriptor();
+    n.readFields(in);
+
+    assertEquals("TestTable", n.getNameAsString());
+    assertEquals(servers, n.getServers());
+    assertEquals(true, n.isWALDisabled());
+    assertEquals(1, n.getColumnFamilies().length);
+    assertEquals(Long.MAX_VALUE -1, n.getMaxFileSize());
+
+  }
+
+  @Test
+  public void testWriteAndReadFieldsWithoutServerSet() throws Exception {
+    HTableDescriptor htd = new HTableDescriptor("TestTable");
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream out = new DataOutputStream(baos);
+    htd.write(out);
+
+    byte[] buf = baos.toByteArray();
+
+    DataInputBuffer in = new DataInputBuffer();
+    in.reset(buf, 0, buf.length);
+
+    HTableDescriptor n = new HTableDescriptor();
+    n.readFields(in);
+
+    assertEquals("TestTable", n.getNameAsString());
+    assertEquals(null, n.getServers());
+  }
+
+  @Test
+  public void testSetServers() throws Exception {
+    HTableDescriptor htd = new HTableDescriptor("TestTable");
+    Set<HServerAddress> servers = new HashSet<>();
+
+    servers.add(new HServerAddress(java.net.InetAddress.getLocalHost().getHostName(),0));
+    htd.setServers(servers);
+
+    assertEquals(servers, htd.getServers());
+
+    htd.setServers(null);
+    assertEquals(null, htd.getServers());
+  }
+
+
+  @Test(expected=IllegalArgumentException.class)
+  public void testSetServersThrowErrorNoServers() throws Exception {
+    HTableDescriptor htd = new HTableDescriptor("testThrowErrorNoServers");
+    htd.addFamily(new HColumnDescriptor("d"));
+    htd.setServers(new HashSet<HServerAddress>());
+  }
+}

Copied: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java
(from r1584179, hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java)
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java?p2=hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java&p1=hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java&r1=1584179&r2=1584180&rev=1584180&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestCreateTableOnServers.java
Wed Apr  2 21:03:34 2014
@@ -43,18 +43,20 @@ import org.junit.Test;
 
 /**
  * Create table only on a set of regionservers instead of whole cluster
- * JIRA: https://issues.apache.org/jira/browse/HBASE-10425
+ * JIRA:
  *
+ * https://issues.apache.org/jira/browse/HBASE-10425
+ * https://issues.apache.org/jira/browse/HBASE-10865
  */
-public class TestHackyCreateTable {
+public class TestCreateTableOnServers {
   final Log LOG = LogFactory.getLog(getClass());
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private HBaseAdmin admin;
-  private static final int NUM_REGION_SERVER = 10;
+  private static final int NUM_REGION_SERVER = 7;
   /**
    * Number of regionservers that we will remove in the test from {@link #NUM_REGION_SERVER}
    */
-  private static final int NUM_REMOVED_RS = 7;
+  private static final int NUM_REMOVED_RS = 4;
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.getConfiguration().set(HConstants.LOAD_BALANCER_IMPL,
@@ -98,26 +100,28 @@ public class TestHackyCreateTable {
     };
     HTableDescriptor desc = new HTableDescriptor(tableName);
     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
-    List<HServerAddress> allServers = TEST_UTIL.getHBaseCluster().getRegionServers();
-    System.out.println("initial number of servers: " + allServers.size());
+    List<HServerAddress> keptServers = TEST_UTIL.getHBaseCluster().getRegionServers();
+    System.out.println("initial number of servers: " + keptServers.size());
     // remove some regionservers
     for (int i = 0; i < NUM_REMOVED_RS; i++) {
-      allServers.remove(0);
+      keptServers.remove(0);
     }
     // create a table on the remaining regionservers
     // refresh Admin to pickup new configuration about using favored nodes
-    admin.createTable(desc, splitKeys, allServers);
+    desc.setServers(keptServers);
+    admin.createTable(desc, splitKeys);
     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
     Map<HRegionInfo,HServerAddress> regions = ht.getRegionsInfo();
     int expectedRegions = splitKeys.length + 1;
     assertEquals("Tried to create " + expectedRegions + " regions " +
-        "but only found " + regions.size(),
-        expectedRegions, regions.size());
+            "but only found " + regions.size(),
+        expectedRegions, regions.size()
+    );
      //verify if we use only a subset of the regionservers
     Set<HServerAddress> serversForTable = new HashSet<HServerAddress>();
     serversForTable.addAll(regions.values());
-    assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, allServers.size());
-    assertEquals(allServers.size(), serversForTable.size());
+    assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, keptServers.size());
+    assertEquals(keptServers.size(), serversForTable.size());
 
     // create one more table only on allServers
     tableName = Bytes.toBytes("testCreateTableWithRegionsOnServers2");
@@ -127,7 +131,8 @@ public class TestHackyCreateTable {
     };
     desc = new HTableDescriptor(tableName);
     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
-    admin.createTable(desc, splitKeys2, allServers);
+    desc.setServers(keptServers);
+    admin.createTable(desc, splitKeys2);
     ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
     regions = ht.getRegionsInfo();
     expectedRegions = splitKeys2.length + 1;
@@ -137,7 +142,24 @@ public class TestHackyCreateTable {
      //verify if we use only a subset of the regionservers
     serversForTable = new HashSet<HServerAddress>();
     serversForTable.addAll(regions.values());
-    assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, allServers.size());
-    assertEquals(allServers.size(), serversForTable.size());
+    assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, keptServers.size());
+    assertEquals(keptServers.size(), serversForTable.size());
+  }
+
+  /**
+   * Test to make sure that HTD without servers work as expected. And the server list is
not
+   * stored without action from the user.
+   * @throws IOException
+   */
+  @Test
+  public void testCreateTableNoServers() throws IOException {
+    String tn = "globalAssignmentDomainTable";
+    HTableDescriptor htd = new HTableDescriptor(tn);
+    htd.addFamily(new HColumnDescriptor(Bytes.toBytes("d")));
+    admin.createTable(htd);
+
+    HTableDescriptor createdHTD = admin.getTableDescriptor(Bytes.toBytes(tn));
+    assertEquals(null, createdHTD.getServers());
+
   }
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java?rev=1584180&r1=1584179&r2=1584180&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHackyCreateTable.java
Wed Apr  2 21:03:34 2014
@@ -1,143 +0,0 @@
-/**
- * Copyright The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Create table only on a set of regionservers instead of whole cluster
- * JIRA: https://issues.apache.org/jira/browse/HBASE-10425
- *
- */
-public class TestHackyCreateTable {
-  final Log LOG = LogFactory.getLog(getClass());
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private HBaseAdmin admin;
-  private static final int NUM_REGION_SERVER = 10;
-  /**
-   * Number of regionservers that we will remove in the test from {@link #NUM_REGION_SERVER}
-   */
-  private static final int NUM_REMOVED_RS = 7;
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().set(HConstants.LOAD_BALANCER_IMPL,
-        "org.apache.hadoop.hbase.master.RegionManager$AssignmentLoadBalancer");
-    TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
-    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
-    TEST_UTIL.startMiniCluster(NUM_REGION_SERVER);
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    this.admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
-  }
-
-  /**
-   * Create two tables only on specific number of regionservers (subset of all
-   * regionservers). Check after the creation whether the regions are assigned
-   * only to those ones.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testCreateTableOnRegionServers() throws IOException {
-    byte[] tableName = Bytes.toBytes("testCreateTableWithRegionsOnServers");
-    byte[][] splitKeys = {
-        new byte[]{1, 1, 1},
-        new byte[]{2, 2, 2},
-        new byte[]{3, 3, 3},
-        new byte[]{4, 4, 4},
-        new byte[]{5, 5, 5},
-        new byte[]{6, 6, 6},
-        new byte[]{7, 7, 7},
-        new byte[]{8, 8, 8},
-        new byte[]{9, 9, 9},
-    };
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
-    List<HServerAddress> allServers = TEST_UTIL.getHBaseCluster().getRegionServers();
-    System.out.println("initial number of servers: " + allServers.size());
-    // remove some regionservers
-    for (int i = 0; i < NUM_REMOVED_RS; i++) {
-      allServers.remove(0);
-    }
-    // create a table on the remaining regionservers
-    // refresh Admin to pickup new configuration about using favored nodes
-    admin.createTable(desc, splitKeys, allServers);
-    HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    Map<HRegionInfo,HServerAddress> regions = ht.getRegionsInfo();
-    int expectedRegions = splitKeys.length + 1;
-    assertEquals("Tried to create " + expectedRegions + " regions " +
-        "but only found " + regions.size(),
-        expectedRegions, regions.size());
-     //verify if we use only a subset of the regionservers
-    Set<HServerAddress> serversForTable = new HashSet<HServerAddress>();
-    serversForTable.addAll(regions.values());
-    assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, allServers.size());
-    assertEquals(allServers.size(), serversForTable.size());
-
-    // create one more table only on allServers
-    tableName = Bytes.toBytes("testCreateTableWithRegionsOnServers2");
-    byte[][] splitKeys2 = {
-        new byte[]{1, 1, 1},
-        new byte[]{2, 2, 2}
-    };
-    desc = new HTableDescriptor(tableName);
-    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
-    admin.createTable(desc, splitKeys2, allServers);
-    ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
-    regions = ht.getRegionsInfo();
-    expectedRegions = splitKeys2.length + 1;
-    assertEquals("Tried to create " + expectedRegions + " regions " +
-        "but only found " + regions.size(),
-        expectedRegions, regions.size());
-     //verify if we use only a subset of the regionservers
-    serversForTable = new HashSet<HServerAddress>();
-    serversForTable.addAll(regions.values());
-    assertEquals(NUM_REGION_SERVER - NUM_REMOVED_RS, allServers.size());
-    assertEquals(allServers.size(), serversForTable.size());
-  }
-}

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java?rev=1584180&r1=1584179&r2=1584180&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/swift/TestSwiftSerDe.java
Wed Apr  2 21:03:34 2014
@@ -25,7 +25,11 @@ import static org.junit.Assert.assertTru
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HRegionInfo;
@@ -197,6 +201,28 @@ public class TestSwiftSerDe {
   }
 
   @Test
+  public void testHTDSerDe() throws Exception {
+    List<HColumnDescriptor> columns = new ArrayList<>(1);
+    Map<byte[], byte[]> values = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+    Set<HServerAddress> servers = new HashSet<>(1);
+    servers.add(new HServerAddress(java.net.InetAddress.getLocalHost().getHostName(),0));
+
+    columns.add(new HColumnDescriptor(Bytes.toBytes("d")));
+
+    HTableDescriptor origionalHTD = new HTableDescriptor(
+        Bytes.toBytes("testSerdeName"),
+        columns,
+        values
+    );
+    origionalHTD.setServers(servers);
+
+    byte[] data = Bytes.writeThriftBytes(origionalHTD, HTableDescriptor.class);
+    HTableDescriptor htdCopy = Bytes.readThriftBytes(data, HTableDescriptor.class);
+    assertEquals(origionalHTD, htdCopy);
+    assertEquals(servers, htdCopy.getServers());
+  }
+
+  @Test
   public void testHFileHistogramSerDe() throws Exception {
     byte [] startRow = Bytes.toBytes("testStartRow");
     byte [] endRow = Bytes.toBytes("testEndRow");



Mime
View raw message