incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/4] git commit: Adding record level security as a builtin feature of Blur. More testing needs to be created to consider this task complete, however the current implementation is complete.
Date Fri, 16 Jan 2015 14:34:31 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
deleted file mode 100644
index 79e4b9f..0000000
--- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ /dev/null
@@ -1,926 +0,0 @@
-package org.apache.blur.thrift;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryMXBean;
-import java.lang.management.MemoryUsage;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.blur.MiniCluster;
-import org.apache.blur.TestType;
-import org.apache.blur.analysis.FieldManager;
-import org.apache.blur.command.RunSlowForTesting;
-import org.apache.blur.server.TableContext;
-import org.apache.blur.thirdparty.thrift_0_9_0.TException;
-import org.apache.blur.thrift.generated.Blur;
-import org.apache.blur.thrift.generated.Blur.Iface;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.BlurQuery;
-import org.apache.blur.thrift.generated.BlurQueryStatus;
-import org.apache.blur.thrift.generated.BlurResult;
-import org.apache.blur.thrift.generated.BlurResults;
-import org.apache.blur.thrift.generated.Column;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.ErrorType;
-import org.apache.blur.thrift.generated.Facet;
-import org.apache.blur.thrift.generated.FetchResult;
-import org.apache.blur.thrift.generated.Query;
-import org.apache.blur.thrift.generated.QueryState;
-import org.apache.blur.thrift.generated.Record;
-import org.apache.blur.thrift.generated.RecordMutation;
-import org.apache.blur.thrift.generated.RowMutation;
-import org.apache.blur.thrift.generated.Schema;
-import org.apache.blur.thrift.generated.Selector;
-import org.apache.blur.thrift.generated.ShardState;
-import org.apache.blur.thrift.generated.SortField;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.thrift.generated.TableStats;
-import org.apache.blur.thrift.util.BlurThriftHelper;
-import org.apache.blur.user.User;
-import org.apache.blur.user.UserContext;
-import org.apache.blur.utils.BlurConstants;
-import org.apache.blur.utils.GCWatcher;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.zookeeper.KeeperException;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class BlurClusterTest {
-
-  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest"));
-  private static MiniCluster miniCluster;
-  private static boolean externalProcesses = true;
-
-  private int numberOfDocs = 1000;
-  private String controllerConnectionStr;
-
-  @BeforeClass
-  public static void startCluster() throws IOException {
-    GCWatcher.init(0.60);
-    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
-    File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
-    testDirectory.mkdirs();
-
-    Path directory = new Path(testDirectory.getPath());
-    FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
-    FsAction userAction = dirPermissions.getUserAction();
-    FsAction groupAction = dirPermissions.getGroupAction();
-    FsAction otherAction = dirPermissions.getOtherAction();
-
-    StringBuilder builder = new StringBuilder();
-    builder.append(userAction.ordinal());
-    builder.append(groupAction.ordinal());
-    builder.append(otherAction.ordinal());
-    String dirPermissionNum = builder.toString();
-    System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
-    testDirectory.delete();
-    miniCluster = new MiniCluster();
-    miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, externalProcesses);
-  }
-
-  @AfterClass
-  public static void shutdownCluster() {
-    miniCluster.shutdownBlurCluster();
-  }
-
-  @After
-  public void tearDown() throws BlurException, TException {
-    Iface client = getClient();
-    List<String> tableList = client.tableList();
-    for (String table : tableList) {
-      client.disableTable(table);
-      client.removeTable(table, true);
-    }
-  }
-
-  private Iface getClient() {
-    if (controllerConnectionStr == null) {
-      controllerConnectionStr = miniCluster.getControllerConnectionStr();
-    }
-    return BlurClient.getClient(controllerConnectionStr);
-  }
-
-  @Test
-  public void testEnqueue() throws BlurException, TException, InterruptedException, IOException {
-    String tableName = "testEnqueue";
-    createTable(tableName);
-    Blur.Iface client = getClient();
-
-    long s = System.currentTimeMillis();
-    int count = 10000;
-    for (int i = 0; i < count; i++) {
-      String rowId = UUID.randomUUID().toString();
-      RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId,
-          BlurThriftHelper.newColumn("test", "value"));
-      RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation);
-      client.enqueueMutate(rowMutation);
-    }
-    long e = System.currentTimeMillis();
-    double seconds = (e - s) / 1000.0;
-    double rate = count / seconds;
-    System.out.println("Load row in queue at " + rate + "/s");
-
-    for (int i = 0; i < 60; i++) {
-      TableStats stats = client.tableStats(tableName);
-      long rowCount = stats.getRowCount();
-      if (rowCount == count) {
-        return;
-      }
-      Thread.sleep(1000);
-    }
-    fail("Test failed to load all rows.");
-  }
-
-  @Test
-  public void testBlurQueryWithRowId() throws BlurException, TException, InterruptedException, IOException {
-    String tableName = "testBlurQueryWithRowId";
-    createTable(tableName);
-    loadTable(tableName);
-    Blur.Iface client = getClient();
-    BlurQuery blurQuery = new BlurQuery();
-    Query query = new Query();
-    query.setQuery("*");
-    blurQuery.setQuery(query);
-    BlurResults results1 = client.query(tableName, blurQuery);
-    assertEquals(numberOfDocs, results1.getTotalResults());
-    String id1 = results1.getResults().iterator().next().getFetchResult().getRowResult().getRow().getId();
-
-    blurQuery.setRowId(id1);
-
-    query.setRowQuery(false);
-    BlurResults results2 = client.query(tableName, blurQuery);
-    assertEquals(1, results2.getTotalResults());
-    String id2 = results2.getResults().iterator().next().getFetchResult().getRecordResult().getRowid();
-
-    assertEquals(id1, id2);
-    System.out.println("Finished!");
-  }
-
-  @Test
-  public void testAdminCalls() throws BlurException, TException, IOException, InterruptedException {
-    String tableName = "testAdminCalls";
-    createTable(tableName);
-    loadTable(tableName);
-    Blur.Iface client = getClient();
-    List<String> shardClusterList = client.shardClusterList();
-    assertEquals(1, shardClusterList.size());
-    assertEquals(BlurConstants.DEFAULT, shardClusterList.get(0));
-
-    Map<String, String> shardServerLayout = client.shardServerLayout(tableName);
-    assertEquals(5, shardServerLayout.size());
-
-    Map<String, Map<String, ShardState>> shardServerLayoutState = client.shardServerLayoutState(tableName);
-    assertEquals(5, shardServerLayoutState.size());
-
-    List<String> shardServerList = client.shardServerList(BlurConstants.DEFAULT);
-    assertEquals(3, shardServerList.size());
-  }
-
-  @Test
-  public void testForEmptySchema() throws BlurException, TException, IOException, InterruptedException {
-    String tableName = "testForEmptySchema";
-    createTable(tableName);
-    Blur.Iface client = getClient();
-    Schema schema = client.schema(tableName);
-    Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies();
-    assertTrue(families.isEmpty());
-
-    TableContext tableContext = TableContext.create(client.describe(tableName));
-    FieldManager fieldManager = tableContext.getFieldManager();
-
-    assertTrue(fieldManager.addColumnDefinition("test-family", "test-column", null, false, "string", false, null));
-
-    TableContext.clear();
-    Schema newschema = client.schema(tableName);
-    Map<String, Map<String, ColumnDefinition>> newfamilies = newschema.getFamilies();
-    assertTrue(!newfamilies.isEmpty());
-    int newsize = newfamilies.size();
-    assertEquals(1, newsize);
-  }
-
-  @Test
-  public void testCreateTableWithCustomType() throws IOException, BlurException, TException {
-    Blur.Iface client = getClient();
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setName("test_type");
-    tableDescriptor.setShardCount(1);
-    tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/test_type");
-    tableDescriptor.putToTableProperties("blur.fieldtype.customtype1", TestType.class.getName());
-    client.createTable(tableDescriptor);
-    List<String> tableList = client.tableList();
-    assertTrue(tableList.contains("test_type"));
-
-    client.disableTable("test_type");
-
-    client.enableTable("test_type");
-
-    TableDescriptor describe = client.describe("test_type");
-    Map<String, String> tableProperties = describe.getTableProperties();
-    assertEquals(TestType.class.getName(), tableProperties.get("blur.fieldtype.customtype1"));
-  }
-
-  public void createTable(String tableName) throws BlurException, TException, IOException {
-    Blur.Iface client = getClient();
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setName(tableName);
-    tableDescriptor.setShardCount(5);
-    tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName);
-    client.createTable(tableDescriptor);
-    List<String> tableList = client.tableList();
-    assertTrue(tableList.contains(tableName));
-  }
-
-  public void loadTable(String tableName) throws BlurException, TException, InterruptedException {
-    loadTable(tableName, 1);
-  }
-
-  public void loadTable(String tableName, int pass) throws BlurException, TException, InterruptedException {
-    Iface client = getClient();
-    int maxFacetValue = 100;
-    List<RowMutation> mutations = new ArrayList<RowMutation>();
-    Random random = new Random(1);
-    for (int i = 0; i < numberOfDocs; i++) {
-      String rowId = UUID.randomUUID().toString();
-      RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId,
-          BlurThriftHelper.newColumn("test", "value"),
-          BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue))),
-          BlurThriftHelper.newColumn("facetFixed", "test"));
-      RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation);
-      mutations.add(rowMutation);
-    }
-    ColumnDefinition columnDefinition = new ColumnDefinition();
-    columnDefinition.setFamily("test");
-    columnDefinition.setColumnName("facet");
-    columnDefinition.setFieldLessIndexed(true);
-    columnDefinition.setFieldType("string");
-    columnDefinition.setSortable(true);
-    columnDefinition.setProperties(new HashMap<String, String>());
-    client.addColumnDefinition(tableName, columnDefinition);
-    long s = System.nanoTime();
-    client.mutateBatch(mutations);
-    long e = System.nanoTime();
-    System.out.println("mutateBatch took [" + (e - s) / 1000000.0 + "]");
-    BlurQuery blurQueryRow = new BlurQuery();
-    Query queryRow = new Query();
-    queryRow.setQuery("test.test:value");
-    blurQueryRow.setQuery(queryRow);
-    blurQueryRow.setUseCacheIfPresent(false);
-    blurQueryRow.setCacheResult(false);
-    BlurResults resultsRow = client.query(tableName, blurQueryRow);
-    assertRowResults(resultsRow);
-    assertEquals(numberOfDocs * pass, resultsRow.getTotalResults());
-
-    BlurQuery blurQueryRecord = new BlurQuery();
-    Query queryRecord = new Query();
-    queryRecord.rowQuery = false;
-    queryRecord.setQuery("test.test:value");
-    blurQueryRecord.setQuery(queryRecord);
-    BlurResults resultsRecord = client.query(tableName, blurQueryRecord);
-    assertRecordResults(resultsRecord);
-    assertEquals(numberOfDocs * pass, resultsRecord.getTotalResults());
-
-    Schema schema = client.schema(tableName);
-    assertFalse(schema.getFamilies().isEmpty());
-  }
-
-  @Test
-  public void testQueryWithSelector() throws BlurException, TException, IOException, InterruptedException {
-    final String tableName = "testQueryWithSelector";
-    createTable(tableName);
-    loadTable(tableName);
-    Iface client = getClient();
-    BlurQuery blurQueryRow = new BlurQuery();
-    Query queryRow = new Query();
-    queryRow.setQuery("test.test:value");
-    blurQueryRow.setQuery(queryRow);
-    blurQueryRow.setUseCacheIfPresent(false);
-    blurQueryRow.setCacheResult(false);
-    blurQueryRow.setSelector(new Selector());
-
-    BlurResults resultsRow = client.query(tableName, blurQueryRow);
-    // assertRowResults(resultsRow);
-    assertEquals(numberOfDocs, resultsRow.getTotalResults());
-
-    for (BlurResult blurResult : resultsRow.getResults()) {
-      System.out.println(blurResult);
-    }
-
-  }
-
-  @Test
-  public void testSortedQueryWithSelector() throws BlurException, TException, IOException, InterruptedException {
-    final String tableName = "testSortedQueryWithSelector";
-    createTable(tableName);
-    loadTable(tableName);
-
-    Iface client = getClient();
-
-    BlurQuery blurQueryRow = new BlurQuery();
-    Query queryRow = new Query();
-    queryRow.setQuery("test.test:value");
-    queryRow.setRowQuery(false);
-    blurQueryRow.setQuery(queryRow);
-    blurQueryRow.addToSortFields(new SortField("test", "facet", false));
-
-    blurQueryRow.setUseCacheIfPresent(false);
-    blurQueryRow.setCacheResult(false);
-    Selector selector = new Selector();
-    selector.setRecordOnly(true);
-    blurQueryRow.setSelector(selector);
-
-    BlurResults resultsRow = client.query(tableName, blurQueryRow);
-    long totalResults = resultsRow.getTotalResults();
-
-    assertEquals(numberOfDocs, resultsRow.getTotalResults());
-
-    String lastValue = null;
-    long totalFetched = 0;
-    do {
-      for (BlurResult blurResult : resultsRow.getResults()) {
-        FetchResult fetchResult = blurResult.getFetchResult();
-        Record record = fetchResult.getRecordResult().getRecord();
-        if (lastValue == null) {
-          lastValue = getColumnValue(record, "facet");
-        } else {
-          String currentValue = getColumnValue(record, "facet");
-          if (currentValue.compareTo(lastValue) < 0) {
-            fail("Current Value of [" + currentValue + "] can not be less than lastValue of [" + lastValue + "]");
-          }
-          lastValue = currentValue;
-        }
-        totalFetched++;
-      }
-      int size = resultsRow.getResults().size();
-      totalResults -= size;
-      if (totalResults > 0) {
-        blurQueryRow.setStart(blurQueryRow.getStart() + size);
-        resultsRow = client.query(tableName, blurQueryRow);
-      }
-    } while (totalResults > 0);
-    assertEquals(numberOfDocs, totalFetched);
-  }
-
-  private String getColumnValue(Record record, String columnName) {
-    for (Column col : record.getColumns()) {
-      if (col.getName().equals(columnName)) {
-        return col.getValue();
-      }
-    }
-    return null;
-  }
-
-  // @Test
-  public void testQueryWithSelectorForDeepPagingPerformance() throws BlurException, TException, IOException,
-      InterruptedException {
-    final String tableName = "testQueryWithSelectorForDeepPagingPerformance";
-    createTable(tableName);
-    int passes = 10;
-    for (int i = 1; i <= passes; i++) {
-      loadTable(tableName, i);
-    }
-    Iface client = getClient();
-    BlurQuery blurQueryRow = new BlurQuery();
-    Query queryRow = new Query();
-    queryRow.setQuery("test.test:value");
-    blurQueryRow.setQuery(queryRow);
-    blurQueryRow.setUseCacheIfPresent(false);
-    blurQueryRow.setCacheResult(false);
-    blurQueryRow.setSelector(new Selector());
-
-    long start = System.nanoTime();
-    int position = 0;
-    do {
-      blurQueryRow.setStart(position);
-      long s = System.nanoTime();
-      BlurResults resultsRow = client.query(tableName, blurQueryRow);
-      long e = System.nanoTime();
-      System.out.println("RUNNING QUERY.... starting at [" + position + "] took [" + (e - s) / 1000000.0 + " ms]");
-      // assertRowResults(resultsRow);
-      assertEquals(numberOfDocs * passes, resultsRow.getTotalResults());
-
-      for (BlurResult blurResult : resultsRow.getResults()) {
-        System.out.println(blurResult);
-        position++;
-      }
-    } while (position < numberOfDocs * passes);
-    long end = System.nanoTime();
-    System.out.println((end - start) / 1000000.0);
-  }
-
-  @Test
-  public void testQueryWithFacets() throws BlurException, TException, IOException, InterruptedException {
-    final String tableName = "testQueryWithFacets";
-    createTable(tableName);
-    loadTable(tableName);
-    Iface client = getClient();
-    BlurQuery blurQueryRow = new BlurQuery();
-    Query queryRow = new Query();
-    // queryRow.setQuery("test.test:value");
-    queryRow.setQuery("*");
-    blurQueryRow.setQuery(queryRow);
-    blurQueryRow.setUseCacheIfPresent(false);
-    blurQueryRow.setCacheResult(false);
-    blurQueryRow.setSelector(new Selector());
-    for (int i = 0; i < 250; i++) {
-      blurQueryRow.addToFacets(new Facet("test.facet:" + i, Long.MAX_VALUE));
-    }
-
-    BlurResults resultsRow = client.query(tableName, blurQueryRow);
-    assertEquals(numberOfDocs, resultsRow.getTotalResults());
-    System.out.println(resultsRow.getFacetCounts());
-
-    System.out.println();
-
-  }
-
-  @Test
-  public void testBatchFetch() throws BlurException, TException, InterruptedException, IOException {
-    String tableName = "testBatchFetch";
-    createTable(tableName);
-    loadTable(tableName);
-    final Iface client = getClient();
-    List<String> terms = client.terms(tableName, null, "rowid", "", (short) 100);
-
-    List<Selector> selectors = new ArrayList<Selector>();
-    for (String s : terms) {
-      Selector selector = new Selector();
-      selector.setRowId(s);
-      selectors.add(selector);
-    }
-
-    List<FetchResult> fetchRowBatch = client.fetchRowBatch(tableName, selectors);
-    assertEquals(100, fetchRowBatch.size());
-
-    int i = 0;
-    for (FetchResult fetchResult : fetchRowBatch) {
-      assertEquals(terms.get(i), fetchResult.getRowResult().getRow().getId());
-      i++;
-    }
-
-  }
-
-  @Test
-  public void testQueryStatus() throws BlurException, TException, InterruptedException, IOException {
-    final String tableName = "testQueryStatus";
-    createTable(tableName);
-    loadTable(tableName);
-    final Iface client = getClient();
-    final BlurQuery blurQueryRow = new BlurQuery();
-    Query queryRow = new Query();
-    queryRow.setQuery("test.test:value");
-    blurQueryRow.setQuery(queryRow);
-    blurQueryRow.setUseCacheIfPresent(false);
-    blurQueryRow.setCacheResult(false);
-    String uuid = "5678";
-    blurQueryRow.setUuid(uuid);
-    final User user = new User("testuser", new HashMap<String, String>());
-
-    try {
-      setDebugRunSlow(tableName, true);
-      new Thread(new Runnable() {
-        @Override
-        public void run() {
-          try {
-            UserContext.setUser(user);
-            // This call will take several seconds to execute.
-            client.query(tableName, blurQueryRow);
-          } catch (BlurException e) {
-            // e.printStackTrace();
-          } catch (TException e) {
-            e.printStackTrace();
-          }
-        }
-      }).start();
-      Thread.sleep(500);
-      BlurQueryStatus queryStatusById = client.queryStatusById(tableName, uuid);
-      assertEquals(user.getUsername(), queryStatusById.getUser().getUsername());
-      assertEquals(queryStatusById.getState(), QueryState.RUNNING);
-      client.cancelQuery(tableName, uuid);
-    } finally {
-      setDebugRunSlow(tableName, false);
-    }
-  }
-
-  @Test
-  public void testQueryCancel() throws BlurException, TException, InterruptedException, IOException {
-    final String tableName = "testQueryCancel";
-    createTable(tableName);
-    loadTable(tableName);
-    final Iface client = getClient();
-    try {
-      // This will make each collect in the collectors pause 250 ms per collect
-      // call
-      setDebugRunSlow(tableName, true);
-      final BlurQuery blurQueryRow = new BlurQuery();
-      Query queryRow = new Query();
-      queryRow.setQuery("test.test:value");
-      blurQueryRow.setQuery(queryRow);
-      blurQueryRow.setUseCacheIfPresent(false);
-      blurQueryRow.setCacheResult(false);
-      blurQueryRow.setUuid("1234");
-
-      final AtomicReference<BlurException> error = new AtomicReference<BlurException>();
-      final AtomicBoolean fail = new AtomicBoolean();
-
-      new Thread(new Runnable() {
-        @Override
-        public void run() {
-          try {
-            // This call will take several seconds to execute.
-            client.query(tableName, blurQueryRow);
-            fail.set(true);
-          } catch (BlurException e) {
-            error.set(e);
-          } catch (TException e) {
-            e.printStackTrace();
-            fail.set(true);
-          }
-        }
-      }).start();
-      Thread.sleep(500);
-      client.cancelQuery(tableName, blurQueryRow.getUuid());
-      BlurException blurException = pollForError(error, 10, TimeUnit.SECONDS, null, fail, -1);
-      if (fail.get()) {
-        fail("Unknown error, failing test.");
-      }
-      assertEquals(blurException.getErrorType(), ErrorType.QUERY_CANCEL);
-    } finally {
-      setDebugRunSlow(tableName, false);
-    }
-    // Tests that the exitable reader was reset.
-    client.terms(tableName, "test", "facet", null, (short) 100);
-  }
-
-  // @Test
-  public void testBackPressureViaQuery() throws BlurException, TException, InterruptedException, IOException {
-    // This will make each collect in the collectors pause 250 ms per collect
-    // call
-    String tableName = "testAdminCalls";
-    createTable(tableName);
-    loadTable(tableName);
-    try {
-      setDebugRunSlow(tableName, true);
-      runBackPressureViaQuery(tableName);
-      Thread.sleep(1000);
-      System.gc();
-      System.gc();
-      Thread.sleep(1000);
-    } finally {
-      setDebugRunSlow(tableName, false);
-    }
-  }
-
-  private void setDebugRunSlow(String table, boolean flag) throws IOException {
-    RunSlowForTesting runSlowForTesting = new RunSlowForTesting();
-    runSlowForTesting.setRunSlow(flag);
-    runSlowForTesting.setTable(table);
-    runSlowForTesting.run(getClient());
-  }
-
-  private void runBackPressureViaQuery(final String tableName) throws InterruptedException {
-    final Iface client = getClient();
-    final BlurQuery blurQueryRow = new BlurQuery();
-    Query queryRow = new Query();
-    queryRow.setQuery("test.test:value");
-    blurQueryRow.setQuery(queryRow);
-    blurQueryRow.setUseCacheIfPresent(false);
-    blurQueryRow.setCacheResult(false);
-    blurQueryRow.setUuid("1234");
-
-    final AtomicReference<BlurException> error = new AtomicReference<BlurException>();
-    final AtomicBoolean fail = new AtomicBoolean();
-
-    System.gc();
-    System.gc();
-    MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
-    MemoryUsage usage = memoryMXBean.getHeapMemoryUsage();
-    long max = usage.getMax();
-    System.out.println("Max Heap [" + max + "]");
-    long used = usage.getUsed();
-    System.out.println("Used Heap [" + used + "]");
-    long limit = (long) (max * 0.80);
-    System.out.println("Limit Heap [" + limit + "]");
-    long difference = limit - used;
-    int sizeToAllocate = (int) ((int) difference * 0.50);
-    System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize()
-        + "]");
-
-    byte[] bufferToFillHeap = new byte[sizeToAllocate];
-    new Thread(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          // This call will take several seconds to execute.
-          client.query(tableName, blurQueryRow);
-          fail.set(true);
-        } catch (BlurException e) {
-          System.out.println("-------------------");
-          System.out.println("-------------------");
-          System.out.println("-------------------");
-          e.printStackTrace();
-          System.out.println("-------------------");
-          System.out.println("-------------------");
-          System.out.println("-------------------");
-          error.set(e);
-        } catch (TException e) {
-          e.printStackTrace();
-          fail.set(true);
-        }
-      }
-    }).start();
-    Thread.sleep(500);
-    List<byte[]> bufferToPutGcWatcherOverLimitList = new ArrayList<byte[]>();
-    BlurException blurException = pollForError(error, 120, TimeUnit.SECONDS, bufferToPutGcWatcherOverLimitList, fail,
-        (int) (difference / 7));
-    if (fail.get()) {
-      fail("Unknown error, failing test.");
-    }
-    System.out.println(bufferToFillHeap.hashCode());
-    System.out.println(bufferToPutGcWatcherOverLimitList.hashCode());
-    assertEquals(blurException.getErrorType(), ErrorType.BACK_PRESSURE);
-    bufferToPutGcWatcherOverLimitList.clear();
-    bufferToPutGcWatcherOverLimitList = null;
-    bufferToFillHeap = null;
-  }
-
-  private BlurException pollForError(AtomicReference<BlurException> error, long period, TimeUnit timeUnit,
-      List<byte[]> bufferToPutGcWatcherOverLimitList, AtomicBoolean fail, int sizeToAllocate)
-      throws InterruptedException {
-    long s = System.nanoTime();
-    long totalTime = timeUnit.toNanos(period) + s;
-    if (bufferToPutGcWatcherOverLimitList != null) {
-      System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize()
-          + "]");
-      bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]);
-    }
-    while (totalTime > System.nanoTime()) {
-      if (fail.get()) {
-        fail("The query failed.");
-      }
-      BlurException blurException = error.get();
-      if (blurException != null) {
-        return blurException;
-      }
-      Thread.sleep(100);
-      if (bufferToPutGcWatcherOverLimitList != null) {
-        if (getHeapSize() < (getMaxHeapSize() * 0.8)) {
-          System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max ["
-              + getMaxHeapSize() + "]");
-          bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]);
-        } else {
-          System.gc();
-          System.gc();
-          System.out.println("Already allocated enough Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() + "]");
-        }
-      }
-    }
-    return null;
-  }
-
-  private long getHeapSize() {
-    return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
-  }
-
-  private long getMaxHeapSize() {
-    return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
-  }
-
-  @Test
-  public void testTestShardFailover() throws BlurException, TException, InterruptedException, IOException,
-      KeeperException {
-    final String tableName = "testTestShardFailover";
-    createTable(tableName);
-    loadTable(tableName);
-    Iface client = getClient();
-    BlurQuery blurQuery = new BlurQuery();
-    blurQuery.setUseCacheIfPresent(false);
-    Query query = new Query();
-    query.setQuery("test.test:value");
-    blurQuery.setQuery(query);
-    BlurResults results1 = client.query(tableName, blurQuery);
-    assertEquals(numberOfDocs, results1.getTotalResults());
-    assertRowResults(results1);
-
-    miniCluster.killShardServer(1);
-
-    // make sure the WAL syncs
-    Thread.sleep(TimeUnit.SECONDS.toMillis(1));
-
-    // This should block until shards have failed over
-    client.shardServerLayout(tableName);
-
-    assertEquals("We should have lost a node.", 2, client.shardServerList(BlurConstants.DEFAULT).size());
-    assertEquals(numberOfDocs, client.query(tableName, blurQuery).getTotalResults());
-
-    miniCluster.startShards(1, true, externalProcesses);
-    Thread.sleep(TimeUnit.SECONDS.toMillis(1));
-
-    assertEquals("We should have the cluster back where we started.", 3, client.shardServerList(BlurConstants.DEFAULT)
-        .size());
-  }
-
-  @Test
-  public void testTrucateRaceCondition() throws BlurException, TException, IOException, InterruptedException {
-    String tableName = "testTrucateRaceCondition";
-    createTable(tableName);
-    loadTable(tableName);
-    List<Connection> connections = BlurClientManager.getConnections(miniCluster.getControllerConnectionStr());
-    Iface client1 = BlurClient.getClient(connections.get(0));
-    Iface client2 = BlurClient.getClient(connections.get(1));
-    TableDescriptor describe = client1.describe(tableName);
-    client1.disableTable(tableName);
-    client1.removeTable(tableName, true);
-    client1.createTable(describe);
-
-    String rowId = UUID.randomUUID().toString();
-    RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId,
-        BlurThriftHelper.newColumn("test", "value"), BlurThriftHelper.newColumn("facetFixed", "test"));
-    RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation);
-    client2.mutate(rowMutation);
-  }
-
-  @Test
-  public void testTableRemovalWithFieldDefsV1() throws BlurException, TException, IOException, InterruptedException {
-    String tableName = "testTableRemovalWithFieldDefsV1";
-    createTable(tableName);
-    loadTable(tableName);
-
-    String family = "testTableRemovalWithFieldDefs-fam";
-
-    List<Connection> connections = BlurClientManager.getConnections(miniCluster.getControllerConnectionStr());
-    Iface client1 = BlurClient.getClient(connections.get(0));
-    ColumnDefinition columnDefinition = new ColumnDefinition();
-    columnDefinition.setColumnName("col");
-    columnDefinition.setFamily(family);
-    columnDefinition.setFieldLessIndexed(false);
-    columnDefinition.setFieldType("string");
-    columnDefinition.setSortable(false);
-    columnDefinition.setSubColumnName(null);
-    assertTrue(client1.addColumnDefinition(tableName, columnDefinition));
-
-    client1.disableTable(tableName);
-    client1.removeTable(tableName, true);
-
-    createTable(tableName);
-
-    Iface client2 = BlurClient.getClient(connections.get(1));
-    assertFamilyIsNotPresent(tableName, client2, family);
-
-    List<String> shardClusterList = client2.shardClusterList();
-
-    for (String cluster : shardClusterList) {
-      List<String> shardServerList = client2.shardServerList(cluster);
-      for (String shardServer : shardServerList) {
-        Iface client = BlurClient.getClient(shardServer);
-        assertFamilyIsNotPresent(tableName, client, family);
-      }
-    }
-
-  }
-
-  @Test
-  public void testTableRemovalWithFieldDefsV2() throws BlurException, TException, IOException, InterruptedException {
-    String tableName = "testTableRemovalWithFieldDefsV2";
-    createTable(tableName);
-    loadTable(tableName);
-
-    Iface client = getClient();
-    Schema expectedSchema = client.schema(tableName);
-    Set<String> expectedFiles = new TreeSet<String>();
-    TableDescriptor describe = client.describe(tableName);
-
-    {
-      Path path = new Path(describe.getTableUri());
-      FileSystem fileSystem = path.getFileSystem(new Configuration());
-      FileStatus[] listStatus = fileSystem.listStatus(new Path(path, "types"));
-      for (FileStatus fileStatus : listStatus) {
-        System.out.println("Expected " + fileStatus.getPath());
-        expectedFiles.add(fileStatus.getPath().toString());
-      }
-    }
-
-    client.disableTable(tableName);
-    client.removeTable(tableName, true);
-
-    createTable(tableName);
-    loadTable(tableName);
-    Schema actualSchema = client.schema(tableName);
-
-    assertEquals(expectedSchema.getTable(), actualSchema.getTable());
-    Map<String, Map<String, ColumnDefinition>> expectedFamilies = expectedSchema.getFamilies();
-    Map<String, Map<String, ColumnDefinition>> actualFamilies = actualSchema.getFamilies();
-    assertEquals(new TreeSet<String>(expectedFamilies.keySet()), new TreeSet<String>(actualFamilies.keySet()));
-
-    for (String family : expectedFamilies.keySet()) {
-      Map<String, ColumnDefinition> expectedColDefMap = new TreeMap<String, ColumnDefinition>(
-          expectedFamilies.get(family));
-      Map<String, ColumnDefinition> actualColDefMap = new TreeMap<String, ColumnDefinition>(actualFamilies.get(family));
-      assertEquals(expectedColDefMap, actualColDefMap);
-    }
-
-    System.out.println(expectedSchema);
-
-    Set<String> actualFiles = new TreeSet<String>();
-    {
-      Path path = new Path(describe.getTableUri());
-      FileSystem fileSystem = path.getFileSystem(new Configuration());
-      FileStatus[] listStatus = fileSystem.listStatus(new Path(path, "types"));
-      for (FileStatus fileStatus : listStatus) {
-        System.out.println("Actual " + fileStatus.getPath());
-        actualFiles.add(fileStatus.getPath().toString());
-      }
-    }
-    assertEquals(expectedFiles, actualFiles);
-  }
-
-  private void assertFamilyIsNotPresent(String tableName, Iface client, String family) throws BlurException, TException {
-    Schema schema = client.schema(tableName);
-    Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies();
-    assertNull(families.get(family));
-  }
-
-  private void assertRowResults(BlurResults results) {
-    for (BlurResult result : results.getResults()) {
-      assertNull(result.locationId);
-      assertNull(result.fetchResult.recordResult);
-      assertNull(result.fetchResult.rowResult.row.records);
-      assertNotNull(result.fetchResult.rowResult.row.id);
-    }
-  }
-
-  private void assertRecordResults(BlurResults results) {
-    for (BlurResult result : results.getResults()) {
-      assertNull(result.locationId);
-      assertNotNull(result.fetchResult.recordResult);
-      assertNotNull(result.fetchResult.recordResult.rowid);
-      assertNotNull(result.fetchResult.recordResult.record.recordId);
-      assertNotNull(result.fetchResult.recordResult.record.family);
-      assertNull("Not null [" + result.fetchResult.recordResult.record.columns + "]",
-          result.fetchResult.recordResult.record.columns);
-      assertNull(result.fetchResult.rowResult);
-    }
-  }
-
-  @Test
-  public void testCreateDisableAndRemoveTable() throws IOException, BlurException, TException {
-    Iface client = getClient();
-    String tableName = UUID.randomUUID().toString();
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setName(tableName);
-    tableDescriptor.setShardCount(5);
-    tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName);
-
-    for (int i = 0; i < 3; i++) {
-      client.createTable(tableDescriptor);
-      client.disableTable(tableName);
-      client.removeTable(tableName, true);
-    }
-
-    assertFalse(client.tableList().contains(tableName));
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
new file mode 100644
index 0000000..83d5adc
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java
@@ -0,0 +1,965 @@
+package org.apache.blur.thrift;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.blur.MiniCluster;
+import org.apache.blur.TestType;
+import org.apache.blur.analysis.FieldManager;
+import org.apache.blur.command.RunSlowForTesting;
+import org.apache.blur.server.TableContext;
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurQueryStatus;
+import org.apache.blur.thrift.generated.BlurResult;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ErrorType;
+import org.apache.blur.thrift.generated.Facet;
+import org.apache.blur.thrift.generated.FetchResult;
+import org.apache.blur.thrift.generated.Query;
+import org.apache.blur.thrift.generated.QueryState;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.Schema;
+import org.apache.blur.thrift.generated.Selector;
+import org.apache.blur.thrift.generated.ShardState;
+import org.apache.blur.thrift.generated.SortField;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.thrift.generated.TableStats;
+import org.apache.blur.thrift.util.BlurThriftHelper;
+import org.apache.blur.user.User;
+import org.apache.blur.user.UserContext;
+import org.apache.blur.utils.BlurConstants;
+import org.apache.blur.utils.GCWatcher;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public abstract class BlurClusterTestBase {
+
+  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest"));
+  private static MiniCluster miniCluster;
+  private static boolean externalProcesses = false;
+
+  private int numberOfDocs = 1000;
+  private String controllerConnectionStr;
+
+  @BeforeClass
+  public static void startCluster() throws IOException {
+    GCWatcher.init(0.60);
+    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
+    File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
+    testDirectory.mkdirs();
+
+    Path directory = new Path(testDirectory.getPath());
+    FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
+    FsAction userAction = dirPermissions.getUserAction();
+    FsAction groupAction = dirPermissions.getGroupAction();
+    FsAction otherAction = dirPermissions.getOtherAction();
+
+    StringBuilder builder = new StringBuilder();
+    builder.append(userAction.ordinal());
+    builder.append(groupAction.ordinal());
+    builder.append(otherAction.ordinal());
+    String dirPermissionNum = builder.toString();
+    System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
+    testDirectory.delete();
+    miniCluster = new MiniCluster();
+    miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, externalProcesses);
+  }
+
+  @AfterClass
+  public static void shutdownCluster() {
+    miniCluster.shutdownBlurCluster();
+  }
+
+  @Before
+  public void setup() {
+    UserContext.setUser(getUser());
+  }
+
+  protected abstract User getUser();
+
+  @After
+  public void tearDown() throws BlurException, TException {
+    Iface client = getClient();
+    List<String> tableList = client.tableList();
+    for (String table : tableList) {
+      client.disableTable(table);
+      client.removeTable(table, true);
+    }
+  }
+
+  private Iface getClient() {
+    if (controllerConnectionStr == null) {
+      controllerConnectionStr = miniCluster.getControllerConnectionStr();
+    }
+    return BlurClient.getClient(controllerConnectionStr);
+  }
+
+  @Test
+  public void testEnqueue() throws BlurException, TException, InterruptedException, IOException {
+    String tableName = "testEnqueue";
+    createTable(tableName);
+    Blur.Iface client = getClient();
+
+    long s = System.currentTimeMillis();
+    int count = 10000;
+    for (int i = 0; i < count; i++) {
+      String rowId = UUID.randomUUID().toString();
+      RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId,
+          BlurThriftHelper.newColumn("test", "value"));
+      RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation);
+      client.enqueueMutate(mutate(rowMutation));
+    }
+    long e = System.currentTimeMillis();
+    double seconds = (e - s) / 1000.0;
+    double rate = count / seconds;
+    System.out.println("Load row in queue at " + rate + "/s");
+
+    for (int i = 0; i < 60; i++) {
+      TableStats stats = client.tableStats(tableName);
+      long rowCount = stats.getRowCount();
+      if (rowCount == count) {
+        return;
+      }
+      Thread.sleep(1000);
+    }
+    fail("Test failed to load all rows.");
+  }
+
+  protected abstract RowMutation mutate(RowMutation rowMutation);
+
+  @Test
+  public void testBlurQueryWithRowId() throws BlurException, TException, InterruptedException, IOException {
+    String tableName = "testBlurQueryWithRowId";
+    createTable(tableName);
+    loadTable(tableName);
+    Blur.Iface client = getClient();
+    BlurQuery blurQuery = new BlurQuery();
+    Query query = new Query();
+    query.setQuery("*");
+    blurQuery.setQuery(query);
+    BlurResults results1 = client.query(tableName, blurQuery);
+    assertEquals(numberOfDocs, results1.getTotalResults());
+    String id1 = results1.getResults().iterator().next().getFetchResult().getRowResult().getRow().getId();
+
+    blurQuery.setRowId(id1);
+
+    query.setRowQuery(false);
+    BlurResults results2 = client.query(tableName, blurQuery);
+    assertEquals(1, results2.getTotalResults());
+    String id2 = results2.getResults().iterator().next().getFetchResult().getRecordResult().getRowid();
+
+    assertEquals(id1, id2);
+    System.out.println("Finished!");
+  }
+
+  @Test
+  public void testAdminCalls() throws BlurException, TException, IOException, InterruptedException {
+    String tableName = "testAdminCalls";
+    createTable(tableName);
+    loadTable(tableName);
+    Blur.Iface client = getClient();
+    List<String> shardClusterList = client.shardClusterList();
+    assertEquals(1, shardClusterList.size());
+    assertEquals(BlurConstants.DEFAULT, shardClusterList.get(0));
+
+    Map<String, String> shardServerLayout = client.shardServerLayout(tableName);
+    assertEquals(5, shardServerLayout.size());
+
+    Map<String, Map<String, ShardState>> shardServerLayoutState = client.shardServerLayoutState(tableName);
+    assertEquals(5, shardServerLayoutState.size());
+
+    List<String> shardServerList = client.shardServerList(BlurConstants.DEFAULT);
+    assertEquals(3, shardServerList.size());
+  }
+
+  @Test
+  public void testForEmptySchema() throws BlurException, TException, IOException, InterruptedException {
+    String tableName = "testForEmptySchema";
+    createTable(tableName, false);
+    Blur.Iface client = getClient();
+    Schema schema = client.schema(tableName);
+    Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies();
+    assertTrue(families.isEmpty());
+
+    TableContext tableContext = TableContext.create(client.describe(tableName));
+    FieldManager fieldManager = tableContext.getFieldManager();
+
+    assertTrue(fieldManager.addColumnDefinition("test-family", "test-column", null, false, "string", false, null));
+
+    TableContext.clear();
+    Schema newschema = client.schema(tableName);
+    Map<String, Map<String, ColumnDefinition>> newfamilies = newschema.getFamilies();
+    assertTrue(!newfamilies.isEmpty());
+    int newsize = newfamilies.size();
+    assertEquals(1, newsize);
+  }
+
+  @Test
+  public void testCreateTableWithCustomType() throws IOException, BlurException, TException {
+    Blur.Iface client = getClient();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName("test_type");
+    tableDescriptor.setShardCount(1);
+    tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/test_type");
+    tableDescriptor.putToTableProperties("blur.fieldtype.customtype1", TestType.class.getName());
+    setupTableProperties(tableDescriptor);
+    client.createTable(tableDescriptor);
+    postTableCreate(tableDescriptor, client);
+    List<String> tableList = client.tableList();
+    assertTrue(tableList.contains("test_type"));
+
+    client.disableTable("test_type");
+
+    client.enableTable("test_type");
+
+    TableDescriptor describe = client.describe("test_type");
+    Map<String, String> tableProperties = describe.getTableProperties();
+    assertEquals(TestType.class.getName(), tableProperties.get("blur.fieldtype.customtype1"));
+  }
+
+  public void createTable(String tableName) throws BlurException, TException, IOException {
+    createTable(tableName, true);
+  }
+
+  public void createTable(String tableName, boolean applyAcl) throws BlurException, TException, IOException {
+    Blur.Iface client = getClient();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName(tableName);
+    tableDescriptor.setShardCount(5);
+    tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName);
+    setupTableProperties(tableDescriptor);
+    client.createTable(tableDescriptor);
+    if (applyAcl) {
+      postTableCreate(tableDescriptor, client);
+    }
+    List<String> tableList = client.tableList();
+    assertTrue(tableList.contains(tableName));
+  }
+
+  protected abstract void postTableCreate(TableDescriptor tableDescriptor, Iface client);
+
+  protected abstract void setupTableProperties(TableDescriptor tableDescriptor);
+
+  public void loadTable(String tableName) throws BlurException, TException, InterruptedException {
+    loadTable(tableName, 1);
+  }
+
+  public void loadTable(String tableName, int pass) throws BlurException, TException, InterruptedException {
+    Iface client = getClient();
+    int maxFacetValue = 100;
+    List<RowMutation> mutations = new ArrayList<RowMutation>();
+    Random random = new Random(1);
+    for (int i = 0; i < numberOfDocs; i++) {
+      String rowId = UUID.randomUUID().toString();
+      RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId,
+          BlurThriftHelper.newColumn("test", "value"),
+          BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue))),
+          BlurThriftHelper.newColumn("facetFixed", "test"));
+      RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation);
+      mutations.add(rowMutation);
+    }
+    ColumnDefinition columnDefinition = new ColumnDefinition();
+    columnDefinition.setFamily("test");
+    columnDefinition.setColumnName("facet");
+    columnDefinition.setFieldLessIndexed(true);
+    columnDefinition.setFieldType("string");
+    columnDefinition.setSortable(true);
+    columnDefinition.setProperties(new HashMap<String, String>());
+    client.addColumnDefinition(tableName, columnDefinition);
+    long s = System.nanoTime();
+    client.mutateBatch(mutate(mutations));
+    long e = System.nanoTime();
+    System.out.println("mutateBatch took [" + (e - s) / 1000000.0 + "]");
+    BlurQuery blurQueryRow = new BlurQuery();
+    Query queryRow = new Query();
+    queryRow.setQuery("test.test:value");
+    blurQueryRow.setQuery(queryRow);
+    blurQueryRow.setUseCacheIfPresent(false);
+    blurQueryRow.setCacheResult(false);
+    BlurResults resultsRow = client.query(tableName, blurQueryRow);
+    assertRowResults(resultsRow);
+    assertEquals(numberOfDocs * pass, resultsRow.getTotalResults());
+
+    BlurQuery blurQueryRecord = new BlurQuery();
+    Query queryRecord = new Query();
+    queryRecord.rowQuery = false;
+    queryRecord.setQuery("test.test:value");
+    blurQueryRecord.setQuery(queryRecord);
+    BlurResults resultsRecord = client.query(tableName, blurQueryRecord);
+    assertRecordResults(resultsRecord);
+    assertEquals(numberOfDocs * pass, resultsRecord.getTotalResults());
+
+    Schema schema = client.schema(tableName);
+    assertFalse(schema.getFamilies().isEmpty());
+  }
+
+  protected List<RowMutation> mutate(List<RowMutation> mutations) {
+    List<RowMutation> rowMutations = new ArrayList<RowMutation>();
+    for (int i = 0; i < mutations.size(); i++) {
+      rowMutations.add(mutate(mutations.get(i)));
+    }
+    return rowMutations;
+  }
+
+  @Test
+  public void testQueryWithSelector() throws BlurException, TException, IOException, InterruptedException {
+    final String tableName = "testQueryWithSelector";
+    createTable(tableName);
+    loadTable(tableName);
+    Iface client = getClient();
+    BlurQuery blurQueryRow = new BlurQuery();
+    Query queryRow = new Query();
+    queryRow.setQuery("test.test:value");
+    blurQueryRow.setQuery(queryRow);
+    blurQueryRow.setUseCacheIfPresent(false);
+    blurQueryRow.setCacheResult(false);
+    blurQueryRow.setSelector(new Selector());
+
+    BlurResults resultsRow = client.query(tableName, blurQueryRow);
+    // assertRowResults(resultsRow);
+    assertEquals(numberOfDocs, resultsRow.getTotalResults());
+
+    for (BlurResult blurResult : resultsRow.getResults()) {
+      System.out.println(blurResult);
+    }
+
+  }
+
+  @Test
+  public void testSortedQueryWithSelector() throws BlurException, TException, IOException, InterruptedException {
+    final String tableName = "testSortedQueryWithSelector";
+    createTable(tableName);
+    loadTable(tableName);
+
+    Iface client = getClient();
+
+    BlurQuery blurQueryRow = new BlurQuery();
+    Query queryRow = new Query();
+    queryRow.setQuery("test.test:value");
+    queryRow.setRowQuery(false);
+    blurQueryRow.setQuery(queryRow);
+    blurQueryRow.addToSortFields(new SortField("test", "facet", false));
+
+    blurQueryRow.setUseCacheIfPresent(false);
+    blurQueryRow.setCacheResult(false);
+    Selector selector = new Selector();
+    selector.setRecordOnly(true);
+    blurQueryRow.setSelector(selector);
+
+    BlurResults resultsRow = client.query(tableName, blurQueryRow);
+    long totalResults = resultsRow.getTotalResults();
+
+    assertEquals(numberOfDocs, resultsRow.getTotalResults());
+
+    String lastValue = null;
+    long totalFetched = 0;
+    do {
+      for (BlurResult blurResult : resultsRow.getResults()) {
+        FetchResult fetchResult = blurResult.getFetchResult();
+        Record record = fetchResult.getRecordResult().getRecord();
+        if (lastValue == null) {
+          lastValue = getColumnValue(record, "facet");
+        } else {
+          String currentValue = getColumnValue(record, "facet");
+          if (currentValue.compareTo(lastValue) < 0) {
+            fail("Current Value of [" + currentValue + "] can not be less than lastValue of [" + lastValue + "]");
+          }
+          lastValue = currentValue;
+        }
+        totalFetched++;
+      }
+      int size = resultsRow.getResults().size();
+      totalResults -= size;
+      if (totalResults > 0) {
+        blurQueryRow.setStart(blurQueryRow.getStart() + size);
+        resultsRow = client.query(tableName, blurQueryRow);
+      }
+    } while (totalResults > 0);
+    assertEquals(numberOfDocs, totalFetched);
+  }
+
+  private String getColumnValue(Record record, String columnName) {
+    for (Column col : record.getColumns()) {
+      if (col.getName().equals(columnName)) {
+        return col.getValue();
+      }
+    }
+    return null;
+  }
+
+  // @Test
+  public void testQueryWithSelectorForDeepPagingPerformance() throws BlurException, TException, IOException,
+      InterruptedException {
+    final String tableName = "testQueryWithSelectorForDeepPagingPerformance";
+    createTable(tableName);
+    int passes = 10;
+    for (int i = 1; i <= passes; i++) {
+      loadTable(tableName, i);
+    }
+    Iface client = getClient();
+    BlurQuery blurQueryRow = new BlurQuery();
+    Query queryRow = new Query();
+    queryRow.setQuery("test.test:value");
+    blurQueryRow.setQuery(queryRow);
+    blurQueryRow.setUseCacheIfPresent(false);
+    blurQueryRow.setCacheResult(false);
+    blurQueryRow.setSelector(new Selector());
+
+    long start = System.nanoTime();
+    int position = 0;
+    do {
+      blurQueryRow.setStart(position);
+      long s = System.nanoTime();
+      BlurResults resultsRow = client.query(tableName, blurQueryRow);
+      long e = System.nanoTime();
+      System.out.println("RUNNING QUERY.... starting at [" + position + "] took [" + (e - s) / 1000000.0 + " ms]");
+      // assertRowResults(resultsRow);
+      assertEquals(numberOfDocs * passes, resultsRow.getTotalResults());
+
+      for (BlurResult blurResult : resultsRow.getResults()) {
+        System.out.println(blurResult);
+        position++;
+      }
+    } while (position < numberOfDocs * passes);
+    long end = System.nanoTime();
+    System.out.println((end - start) / 1000000.0);
+  }
+
+  @Test
+  public void testQueryWithFacets() throws BlurException, TException, IOException, InterruptedException {
+    final String tableName = "testQueryWithFacets";
+    createTable(tableName);
+    loadTable(tableName);
+    Iface client = getClient();
+    BlurQuery blurQueryRow = new BlurQuery();
+    Query queryRow = new Query();
+    // queryRow.setQuery("test.test:value");
+    queryRow.setQuery("*");
+    blurQueryRow.setQuery(queryRow);
+    blurQueryRow.setUseCacheIfPresent(false);
+    blurQueryRow.setCacheResult(false);
+    blurQueryRow.setSelector(new Selector());
+    for (int i = 0; i < 250; i++) {
+      blurQueryRow.addToFacets(new Facet("test.facet:" + i, Long.MAX_VALUE));
+    }
+
+    BlurResults resultsRow = client.query(tableName, blurQueryRow);
+    assertEquals(numberOfDocs, resultsRow.getTotalResults());
+    System.out.println(resultsRow.getFacetCounts());
+
+    System.out.println();
+
+  }
+
+  @Test
+  public void testBatchFetch() throws BlurException, TException, InterruptedException, IOException {
+    String tableName = "testBatchFetch";
+    createTable(tableName);
+    loadTable(tableName);
+    final Iface client = getClient();
+    List<String> terms = client.terms(tableName, null, "rowid", "", (short) 100);
+
+    List<Selector> selectors = new ArrayList<Selector>();
+    for (String s : terms) {
+      Selector selector = new Selector();
+      selector.setRowId(s);
+      selectors.add(selector);
+    }
+
+    List<FetchResult> fetchRowBatch = client.fetchRowBatch(tableName, selectors);
+    assertEquals(100, fetchRowBatch.size());
+
+    int i = 0;
+    for (FetchResult fetchResult : fetchRowBatch) {
+      assertEquals(terms.get(i), fetchResult.getRowResult().getRow().getId());
+      i++;
+    }
+
+  }
+
+  @Test
+  public void testQueryStatus() throws BlurException, TException, InterruptedException, IOException {
+    final String tableName = "testQueryStatus";
+    createTable(tableName);
+    loadTable(tableName);
+    final Iface client = getClient();
+    final BlurQuery blurQueryRow = new BlurQuery();
+    Query queryRow = new Query();
+    queryRow.setQuery("test.test:value");
+    blurQueryRow.setQuery(queryRow);
+    blurQueryRow.setUseCacheIfPresent(false);
+    blurQueryRow.setCacheResult(false);
+    String uuid = "5678";
+    blurQueryRow.setUuid(uuid);
+    final User user = new User("testuser", getUserAttributes());
+    try {
+      setDebugRunSlow(tableName, true);
+      new Thread(new Runnable() {
+        @Override
+        public void run() {
+          try {
+            UserContext.setUser(user);
+            // This call will take several seconds to execute.
+            client.query(tableName, blurQueryRow);
+          } catch (BlurException e) {
+            // e.printStackTrace();
+          } catch (TException e) {
+            e.printStackTrace();
+          }
+        }
+      }).start();
+      Thread.sleep(500);
+      BlurQueryStatus queryStatusById = client.queryStatusById(tableName, uuid);
+      assertEquals(user.getUsername(), queryStatusById.getUser().getUsername());
+      assertEquals(queryStatusById.getState(), QueryState.RUNNING);
+      client.cancelQuery(tableName, uuid);
+    } finally {
+      setDebugRunSlow(tableName, false);
+    }
+  }
+  
+  protected abstract Map<String, String> getUserAttributes();
+
+  @Test
+  public void testQueryCancel() throws BlurException, TException, InterruptedException, IOException {
+    final String tableName = "testQueryCancel";
+    createTable(tableName);
+    loadTable(tableName);
+    final Iface client = getClient();
+    try {
+      // This will make each collect in the collectors pause 250 ms per collect
+      // call
+      setDebugRunSlow(tableName, true);
+      final BlurQuery blurQueryRow = new BlurQuery();
+      Query queryRow = new Query();
+      queryRow.setQuery("test.test:value");
+      blurQueryRow.setQuery(queryRow);
+      blurQueryRow.setUseCacheIfPresent(false);
+      blurQueryRow.setCacheResult(false);
+      blurQueryRow.setUuid("1234");
+
+      final AtomicReference<BlurException> error = new AtomicReference<BlurException>();
+      final AtomicBoolean fail = new AtomicBoolean();
+      final User user = UserContext.getUser();
+      new Thread(new Runnable() {
+        @Override
+        public void run() {
+          try {
+            UserContext.setUser(user);
+            // This call will take several seconds to execute.
+            client.query(tableName, blurQueryRow);
+            fail.set(true);
+          } catch (BlurException e) {
+            error.set(e);
+          } catch (TException e) {
+            e.printStackTrace();
+            fail.set(true);
+          }
+        }
+      }).start();
+      Thread.sleep(500);
+      client.cancelQuery(tableName, blurQueryRow.getUuid());
+      BlurException blurException = pollForError(error, 10, TimeUnit.SECONDS, null, fail, -1);
+      if (fail.get()) {
+        fail("Unknown error, failing test.");
+      }
+      assertEquals(blurException.getErrorType(), ErrorType.QUERY_CANCEL);
+    } finally {
+      setDebugRunSlow(tableName, false);
+    }
+    // Tests that the exitable reader was reset.
+    client.terms(tableName, "test", "facet", null, (short) 100);
+  }
+
+  // @Test
+  public void testBackPressureViaQuery() throws BlurException, TException, InterruptedException, IOException {
+    // This will make each collect in the collectors pause 250 ms per collect
+    // call
+    String tableName = "testAdminCalls";
+    createTable(tableName);
+    loadTable(tableName);
+    try {
+      setDebugRunSlow(tableName, true);
+      runBackPressureViaQuery(tableName);
+      Thread.sleep(1000);
+      System.gc();
+      System.gc();
+      Thread.sleep(1000);
+    } finally {
+      setDebugRunSlow(tableName, false);
+    }
+  }
+
+  private void setDebugRunSlow(String table, boolean flag) throws IOException {
+    RunSlowForTesting runSlowForTesting = new RunSlowForTesting();
+    runSlowForTesting.setRunSlow(flag);
+    runSlowForTesting.setTable(table);
+    runSlowForTesting.run(getClient());
+  }
+
+  private void runBackPressureViaQuery(final String tableName) throws InterruptedException {
+    final Iface client = getClient();
+    final BlurQuery blurQueryRow = new BlurQuery();
+    Query queryRow = new Query();
+    queryRow.setQuery("test.test:value");
+    blurQueryRow.setQuery(queryRow);
+    blurQueryRow.setUseCacheIfPresent(false);
+    blurQueryRow.setCacheResult(false);
+    blurQueryRow.setUuid("1234");
+
+    final AtomicReference<BlurException> error = new AtomicReference<BlurException>();
+    final AtomicBoolean fail = new AtomicBoolean();
+
+    System.gc();
+    System.gc();
+    MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+    MemoryUsage usage = memoryMXBean.getHeapMemoryUsage();
+    long max = usage.getMax();
+    System.out.println("Max Heap [" + max + "]");
+    long used = usage.getUsed();
+    System.out.println("Used Heap [" + used + "]");
+    long limit = (long) (max * 0.80);
+    System.out.println("Limit Heap [" + limit + "]");
+    long difference = limit - used;
+    int sizeToAllocate = (int) ((int) difference * 0.50);
+    System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize()
+        + "]");
+
+    byte[] bufferToFillHeap = new byte[sizeToAllocate];
+    final User user = UserContext.getUser();
+    new Thread(new Runnable() {
+      @Override
+      public void run() {
+        UserContext.setUser(user);
+        try {
+          // This call will take several seconds to execute.
+          client.query(tableName, blurQueryRow);
+          fail.set(true);
+        } catch (BlurException e) {
+          System.out.println("-------------------");
+          System.out.println("-------------------");
+          System.out.println("-------------------");
+          e.printStackTrace();
+          System.out.println("-------------------");
+          System.out.println("-------------------");
+          System.out.println("-------------------");
+          error.set(e);
+        } catch (TException e) {
+          e.printStackTrace();
+          fail.set(true);
+        }
+      }
+    }).start();
+    Thread.sleep(500);
+    List<byte[]> bufferToPutGcWatcherOverLimitList = new ArrayList<byte[]>();
+    BlurException blurException = pollForError(error, 120, TimeUnit.SECONDS, bufferToPutGcWatcherOverLimitList, fail,
+        (int) (difference / 7));
+    if (fail.get()) {
+      fail("Unknown error, failing test.");
+    }
+    System.out.println(bufferToFillHeap.hashCode());
+    System.out.println(bufferToPutGcWatcherOverLimitList.hashCode());
+    assertEquals(blurException.getErrorType(), ErrorType.BACK_PRESSURE);
+    bufferToPutGcWatcherOverLimitList.clear();
+    bufferToPutGcWatcherOverLimitList = null;
+    bufferToFillHeap = null;
+  }
+
+  private BlurException pollForError(AtomicReference<BlurException> error, long period, TimeUnit timeUnit,
+      List<byte[]> bufferToPutGcWatcherOverLimitList, AtomicBoolean fail, int sizeToAllocate)
+      throws InterruptedException {
+    long s = System.nanoTime();
+    long totalTime = timeUnit.toNanos(period) + s;
+    if (bufferToPutGcWatcherOverLimitList != null) {
+      System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize()
+          + "]");
+      bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]);
+    }
+    while (totalTime > System.nanoTime()) {
+      if (fail.get()) {
+        fail("The query failed.");
+      }
+      BlurException blurException = error.get();
+      if (blurException != null) {
+        return blurException;
+      }
+      Thread.sleep(100);
+      if (bufferToPutGcWatcherOverLimitList != null) {
+        if (getHeapSize() < (getMaxHeapSize() * 0.8)) {
+          System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max ["
+              + getMaxHeapSize() + "]");
+          bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]);
+        } else {
+          System.gc();
+          System.gc();
+          System.out.println("Already allocated enough Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() + "]");
+        }
+      }
+    }
+    return null;
+  }
+
+  private long getHeapSize() {
+    return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
+  }
+
+  private long getMaxHeapSize() {
+    return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+  }
+
+  @Test
+  public void testTestShardFailover() throws BlurException, TException, InterruptedException, IOException,
+      KeeperException {
+    final String tableName = "testTestShardFailover";
+    createTable(tableName);
+    loadTable(tableName);
+    Iface client = getClient();
+    BlurQuery blurQuery = new BlurQuery();
+    blurQuery.setUseCacheIfPresent(false);
+    Query query = new Query();
+    query.setQuery("test.test:value");
+    blurQuery.setQuery(query);
+    BlurResults results1 = client.query(tableName, blurQuery);
+    assertEquals(numberOfDocs, results1.getTotalResults());
+    assertRowResults(results1);
+
+    miniCluster.killShardServer(1);
+
+    // make sure the WAL syncs
+    Thread.sleep(TimeUnit.SECONDS.toMillis(1));
+
+    // This should block until shards have failed over
+    client.shardServerLayout(tableName);
+
+    assertEquals("We should have lost a node.", 2, client.shardServerList(BlurConstants.DEFAULT).size());
+    assertEquals(numberOfDocs, client.query(tableName, blurQuery).getTotalResults());
+
+    miniCluster.startShards(1, true, externalProcesses);
+    Thread.sleep(TimeUnit.SECONDS.toMillis(1));
+
+    assertEquals("We should have the cluster back where we started.", 3, client.shardServerList(BlurConstants.DEFAULT)
+        .size());
+  }
+
+  @Test
+  public void testTrucateRaceCondition() throws BlurException, TException, IOException, InterruptedException {
+    String tableName = "testTrucateRaceCondition";
+    createTable(tableName);
+    loadTable(tableName);
+    List<Connection> connections = BlurClientManager.getConnections(miniCluster.getControllerConnectionStr());
+    Iface client1 = BlurClient.getClient(connections.get(0));
+    Iface client2 = BlurClient.getClient(connections.get(1));
+    TableDescriptor describe = client1.describe(tableName);
+    setupTableProperties(describe);
+    client1.disableTable(tableName);
+    client1.removeTable(tableName, true);
+    client1.createTable(describe);
+    postTableCreate(describe, client1);
+
+    String rowId = UUID.randomUUID().toString();
+    RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId,
+        BlurThriftHelper.newColumn("test", "value"), BlurThriftHelper.newColumn("facetFixed", "test"));
+    RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation);
+    client2.mutate(mutate(rowMutation));
+  }
+
+  @Test
+  public void testTableRemovalWithFieldDefsV1() throws BlurException, TException, IOException, InterruptedException {
+    String tableName = "testTableRemovalWithFieldDefsV1";
+    createTable(tableName);
+    loadTable(tableName);
+
+    String family = "testTableRemovalWithFieldDefs-fam";
+
+    List<Connection> connections = BlurClientManager.getConnections(miniCluster.getControllerConnectionStr());
+    Iface client1 = BlurClient.getClient(connections.get(0));
+    ColumnDefinition columnDefinition = new ColumnDefinition();
+    columnDefinition.setColumnName("col");
+    columnDefinition.setFamily(family);
+    columnDefinition.setFieldLessIndexed(false);
+    columnDefinition.setFieldType("string");
+    columnDefinition.setSortable(false);
+    columnDefinition.setSubColumnName(null);
+    assertTrue(client1.addColumnDefinition(tableName, columnDefinition));
+
+    client1.disableTable(tableName);
+    client1.removeTable(tableName, true);
+
+    createTable(tableName);
+
+    Iface client2 = BlurClient.getClient(connections.get(1));
+    assertFamilyIsNotPresent(tableName, client2, family);
+
+    List<String> shardClusterList = client2.shardClusterList();
+
+    for (String cluster : shardClusterList) {
+      List<String> shardServerList = client2.shardServerList(cluster);
+      for (String shardServer : shardServerList) {
+        Iface client = BlurClient.getClient(shardServer);
+        assertFamilyIsNotPresent(tableName, client, family);
+      }
+    }
+
+  }
+
+  @Test
+  public void testTableRemovalWithFieldDefsV2() throws BlurException, TException, IOException, InterruptedException {
+    String tableName = "testTableRemovalWithFieldDefsV2";
+    createTable(tableName);
+    loadTable(tableName);
+
+    Iface client = getClient();
+    Schema expectedSchema = client.schema(tableName);
+    Set<String> expectedFiles = new TreeSet<String>();
+    TableDescriptor describe = client.describe(tableName);
+
+    {
+      Path path = new Path(describe.getTableUri());
+      FileSystem fileSystem = path.getFileSystem(new Configuration());
+      FileStatus[] listStatus = fileSystem.listStatus(new Path(path, "types"));
+      for (FileStatus fileStatus : listStatus) {
+        System.out.println("Expected " + fileStatus.getPath());
+        expectedFiles.add(fileStatus.getPath().toString());
+      }
+    }
+
+    client.disableTable(tableName);
+    client.removeTable(tableName, true);
+
+    createTable(tableName);
+    loadTable(tableName);
+    Schema actualSchema = client.schema(tableName);
+
+    assertEquals(expectedSchema.getTable(), actualSchema.getTable());
+    Map<String, Map<String, ColumnDefinition>> expectedFamilies = expectedSchema.getFamilies();
+    Map<String, Map<String, ColumnDefinition>> actualFamilies = actualSchema.getFamilies();
+    assertEquals(new TreeSet<String>(expectedFamilies.keySet()), new TreeSet<String>(actualFamilies.keySet()));
+
+    for (String family : expectedFamilies.keySet()) {
+      Map<String, ColumnDefinition> expectedColDefMap = new TreeMap<String, ColumnDefinition>(
+          expectedFamilies.get(family));
+      Map<String, ColumnDefinition> actualColDefMap = new TreeMap<String, ColumnDefinition>(actualFamilies.get(family));
+      assertEquals(expectedColDefMap, actualColDefMap);
+    }
+
+    System.out.println(expectedSchema);
+
+    Set<String> actualFiles = new TreeSet<String>();
+    {
+      Path path = new Path(describe.getTableUri());
+      FileSystem fileSystem = path.getFileSystem(new Configuration());
+      FileStatus[] listStatus = fileSystem.listStatus(new Path(path, "types"));
+      for (FileStatus fileStatus : listStatus) {
+        System.out.println("Actual " + fileStatus.getPath());
+        actualFiles.add(fileStatus.getPath().toString());
+      }
+    }
+    assertEquals(expectedFiles, actualFiles);
+  }
+
+  private void assertFamilyIsNotPresent(String tableName, Iface client, String family) throws BlurException, TException {
+    Schema schema = client.schema(tableName);
+    Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies();
+    assertNull(families.get(family));
+  }
+
+  private void assertRowResults(BlurResults results) {
+    for (BlurResult result : results.getResults()) {
+      assertNull(result.locationId);
+      assertNull(result.fetchResult.recordResult);
+      assertNull(result.fetchResult.rowResult.row.records);
+      assertNotNull(result.fetchResult.rowResult.row.id);
+    }
+  }
+
+  private void assertRecordResults(BlurResults results) {
+    for (BlurResult result : results.getResults()) {
+      assertNull(result.locationId);
+      assertNotNull(result.fetchResult.recordResult);
+      assertNotNull(result.fetchResult.recordResult.rowid);
+      assertNotNull(result.fetchResult.recordResult.record.recordId);
+      assertNotNull(result.fetchResult.recordResult.record.family);
+      assertNull("Not null [" + result.fetchResult.recordResult.record.columns + "]",
+          result.fetchResult.recordResult.record.columns);
+      assertNull(result.fetchResult.rowResult);
+    }
+  }
+
+  @Test
+  public void testCreateDisableAndRemoveTable() throws IOException, BlurException, TException {
+    Iface client = getClient();
+    String tableName = UUID.randomUUID().toString();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName(tableName);
+    tableDescriptor.setShardCount(5);
+    tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName);
+    setupTableProperties(tableDescriptor);
+    for (int i = 0; i < 3; i++) {
+      client.createTable(tableDescriptor);
+      postTableCreate(tableDescriptor, client);
+      client.disableTable(tableName);
+      client.removeTable(tableName, true);
+    }
+
+    assertFalse(client.tableList().contains(tableName));
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java
new file mode 100644
index 0000000..1504f01
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.thrift;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.user.User;
+
+public class BlurClusterTestNoSecurity extends BlurClusterTestBase {
+
+  @Override
+  protected void setupTableProperties(TableDescriptor tableDescriptor) {
+    // do nothing
+  }
+
+  @Override
+  protected RowMutation mutate(RowMutation rowMutation) {
+    return rowMutation;
+  }
+
+  @Override
+  protected void postTableCreate(TableDescriptor tableDescriptor, Iface client) {
+
+  }
+
+  @Override
+  protected User getUser() {
+    return null;
+  }
+
+  @Override
+  protected Map<String, String> getUserAttributes() {
+    return new HashMap<String, String>();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java
new file mode 100644
index 0000000..b669406
--- /dev/null
+++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java
@@ -0,0 +1,83 @@
+/**
+ing ok * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.thrift;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.user.User;
+import org.apache.blur.utils.BlurConstants;
+
+public class BlurClusterTestSecurity extends BlurClusterTestBase {
+
+  private static final String DISCOVER = "discover";
+  private static final String READ = "read";
+
+  @Override
+  protected void setupTableProperties(TableDescriptor tableDescriptor) {
+    tableDescriptor.putToTableProperties(BlurConstants.BLUR_RECORD_SECURITY, "true");
+  }
+
+  @Override
+  protected RowMutation mutate(RowMutation rowMutation) {
+    List<RecordMutation> mutations = rowMutation.getRecordMutations();
+    for (RecordMutation mutation : mutations) {
+      Record record = mutation.getRecord();
+      record.addToColumns(new Column("acl-read", READ));
+      record.addToColumns(new Column("acl-discover", DISCOVER));
+    }
+    return rowMutation;
+  }
+
+  @Override
+  protected void postTableCreate(TableDescriptor tableDescriptor, Iface client) {
+    String name = tableDescriptor.getName();
+    try {
+      client.addColumnDefinition(name, new ColumnDefinition("test", "acl-read", null, false, "acl-read", null, false));
+      client.addColumnDefinition(name, new ColumnDefinition("test", "acl-discover", null, false, "acl-discover", null,
+          false));
+    } catch (BlurException e) {
+      throw new RuntimeException(e);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  protected User getUser() {
+    return new User("testuser", getUserAttributes());
+  }
+
+  @Override
+  protected Map<String, String> getUserAttributes() {
+    Map<String, String> attributes = new HashMap<String, String>();
+    attributes.put(BlurConstants.ACL_READ, READ);
+    attributes.put(BlurConstants.ACL_DISCOVER, DISCOVER);
+    return attributes;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-query/pom.xml
----------------------------------------------------------------------
diff --git a/blur-query/pom.xml b/blur-query/pom.xml
index 8be6d32..03eb6f3 100644
--- a/blur-query/pom.xml
+++ b/blur-query/pom.xml
@@ -63,11 +63,6 @@
 			<version>${lucene.version}</version>
 		</dependency>
 		<dependency>
-			<groupId>com.github.amccurry</groupId>
-			<artifactId>lucene-document-security</artifactId>
-			<version>0.1.0</version>
-		</dependency>
-		<dependency>
 			<groupId>commons-cli</groupId>
 			<artifactId>commons-cli</artifactId>
 			<version>${commons-cli.version}</version>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseable.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseable.java b/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseable.java
new file mode 100644
index 0000000..31d4004
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseable.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.search;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.Similarity;
+
+public interface IndexSearcherCloseable extends Closeable {
+
+  IndexReader getIndexReader();
+
+  TopDocs search(Query query, int i) throws IOException;
+
+  Query rewrite(Query query) throws IOException;
+
+  void search(Query query, Collector collector) throws IOException;
+
+  void setSimilarity(Similarity similarity);
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseableUtil.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseableUtil.java b/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseableUtil.java
new file mode 100644
index 0000000..973a5ba
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseableUtil.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.search;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.Similarity;
+
+public class IndexSearcherCloseableUtil {
+
+  public static IndexSearcherCloseable wrap(final IndexSearcher searcher) {
+    return new IndexSearcherCloseable() {
+
+      @Override
+      public void close() throws IOException {
+
+      }
+
+      @Override
+      public void setSimilarity(Similarity similarity) {
+        searcher.setSimilarity(similarity);
+      }
+
+      @Override
+      public void search(Query query, Collector collector) throws IOException {
+        searcher.search(query, collector);
+      }
+
+      @Override
+      public TopDocs search(Query query, int n) throws IOException {
+        return searcher.search(query, n);
+      }
+
+      @Override
+      public Query rewrite(Query query) throws IOException {
+        return searcher.rewrite(query);
+      }
+
+      @Override
+      public IndexReader getIndexReader() {
+        return searcher.getIndexReader();
+      }
+    };
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java b/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
index 3bb2d74..8089346 100644
--- a/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
+++ b/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java
@@ -34,7 +34,6 @@ import org.apache.blur.utils.BlurIterable;
 import org.apache.blur.utils.BlurIterator;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Sort;
@@ -53,7 +52,7 @@ public class IterablePaging implements BlurIterable<ScoreDoc, BlurException> {
   private static final boolean DISABLED = true;
 
   private final DeepPagingCache _deepPagingCache;
-  private final IndexSearcher _searcher;
+  private final IndexSearcherCloseable _searcher;
   private final Query _query;
   private final AtomicBoolean _running;
   private final int _numHitsToCollect;
@@ -66,7 +65,7 @@ public class IterablePaging implements BlurIterable<ScoreDoc, BlurException> {
   private int skipTo;
   private int gather = -1;
 
-  public IterablePaging(AtomicBoolean running, IndexSearcher searcher, Query query, int numHitsToCollect,
+  public IterablePaging(AtomicBoolean running, IndexSearcherCloseable searcher, Query query, int numHitsToCollect,
       TotalHitsRef totalHitsRef, ProgressRef progressRef, boolean runSlow, Sort sort, DeepPagingCache deepPagingCache)
       throws BlurException {
     _deepPagingCache = deepPagingCache;


Mime
View raw message