incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [31/51] [partial] Initial repackage to org.apache.
Date Mon, 03 Sep 2012 03:17:13 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateInsertQueryRepeating.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateInsertQueryRepeating.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateInsertQueryRepeating.java
new file mode 100644
index 0000000..053506f
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateInsertQueryRepeating.java
@@ -0,0 +1,241 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+import org.apache.blur.thrift.generated.SimpleQuery;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+import static org.apache.blur.utils.BlurUtil.*;
+
+/**
+ * Tests a lot of things, mainly connecting to a blur cluster and slamming a
+ * bunch of rows in before querying for them. I like to use it as a load test.
+ * 
+ */
+public class CreateInsertQueryRepeating {
+
+  private DecimalFormat df = new DecimalFormat("#,###,000.00");
+  private static final char[] symbols = new char[36];
+
+  static {
+    for (int idx = 0; idx < 10; ++idx)
+      symbols[idx] = (char) ('0' + idx);
+    for (int idx = 10; idx < 36; ++idx)
+      symbols[idx] = (char) ('a' + idx - 10);
+  }
+
+  private String table = "test1";
+  private String host = "localhost";
+  private Iface client = null;
+
+  public CreateInsertQueryRepeating(String host, String table) throws BlurException, TException, IOException {
+    this.host = host;
+    this.table = table;
+
+    // init
+    String connectionStr = host + ":40010";
+    String cluster = "default";
+    client = BlurClient.getClient(connectionStr);
+
+    List<String> clusterList = client.shardClusterList();
+    if (clusterList != null && clusterList.size() > 0)
+      cluster = clusterList.get(0);
+    else
+      throw new IOException("cannot find a cluster to use :(");
+
+    System.out.println("using cluster: " + cluster);
+
+    List<String> tableList = client.tableList();
+    if (tableList == null || !tableList.contains(table))
+      createTable(client, table, cluster);
+    else
+      System.out.println("table existed, did not create.");
+  }
+
+  private final Random random = new Random();
+
+  public String randomString(int length) {
+    char[] buf = new char[length];
+
+    for (int idx = 0; idx < buf.length; ++idx)
+      buf[idx] = symbols[random.nextInt(symbols.length)];
+    return new String(buf);
+  }
+
+  public void getClusters(Iface client) {
+    try {
+      List<String> shardClusterList = client.shardClusterList();
+      for (String cluster : shardClusterList)
+        System.out.println("cluster: " + cluster);
+    } catch (BlurException e) {
+      e.printStackTrace();
+    } catch (TException e) {
+      e.printStackTrace();
+    }
+  }
+
+  public void createTable(Iface client, String tableName, String cluster) throws BlurException, TException {
+    TableDescriptor td = new TableDescriptor();
+    td.analyzerDefinition = new AnalyzerDefinition();
+
+    td.name = tableName;
+    // TODO: doc doesnt say required, yet it barfs without it?
+    td.cluster = cluster == null ? "default" : cluster;
+    // auto enable table
+    td.isEnabled = true;
+
+    // 1 shard per server :)
+    td.shardCount = client.shardServerList(cluster).size();
+    td.readOnly = false;
+    // TODO: hardcodes bad, assuming NN on same node as BC
+    td.tableUri = "hdfs://" + host + ":8020/" + tableName;
+    client.createTable(td);
+    System.out.println("table created");
+  }
+
+  /**
+   * @param args
+   * @throws TException
+   * @throws BlurException
+   * @throws IOException
+   */
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    String host = "localhost";
+    String table = "test1";
+
+    if (args != null) {
+      if (args.length >= 1)
+        host = args[0];
+      if (args.length == 2)
+        table = args[1];
+    }
+
+    CreateInsertQueryRepeating test = new CreateInsertQueryRepeating(host, table);
+
+    // System.out.println("Testing joins real quick");
+    // test.testJoin();
+    // System.out.println("test done");
+
+    System.out.println("Starting load");
+    test.loadupTable(100);
+    System.out.println("Finshed load");
+
+    System.out.println("query time!");
+    test.queryTable(50000);
+    System.out.println("query done!");
+
+    System.exit(0);
+  }
+
+  @SuppressWarnings("unused")
+  private void testJoin() throws BlurException, TException {
+    RowMutation mutation = new RowMutation();
+    mutation.table = table;
+    mutation.waitToBeVisible = true;
+    mutation.rowId = "row1";
+    mutation.addToRecordMutations(newRecordMutation("cf1", "recordid1", newColumn("col1", "value1")));
+    mutation.addToRecordMutations(newRecordMutation("cf1", "recordid2", newColumn("col2", "value2")));
+    mutation.rowMutationType = RowMutationType.REPLACE_ROW;
+    client.mutate(mutation);
+
+    List<String> joinTest = new ArrayList<String>();
+    joinTest.add("+cf1.col1:value1");
+    joinTest.add("+cf1.col2:value2");
+    joinTest.add("+cf1.col1:value1 +cf1.col2:value2");
+    joinTest.add("+(+cf1.col1:value1 nocf.nofield:somevalue) +(+cf1.col2.value2 nocf.nofield:somevalue)");
+    joinTest.add("+(+cf1.col1:value1) +(cf1.bla:bla +cf1.col2.value2)");
+
+    for (String q : joinTest)
+      System.out.println(q + " hits: " + hits(client, table, q, true));
+  }
+
+  private static long hits(Iface client, String table, String queryStr, boolean superQuery) throws BlurException, TException {
+    BlurQuery bq = new BlurQuery();
+    SimpleQuery sq = new SimpleQuery();
+    sq.queryStr = queryStr;
+    sq.superQueryOn = superQuery;
+    bq.simpleQuery = sq;
+    BlurResults query = client.query(table, bq);
+    return query.totalResults;
+  }
+
+  // really only useful against the table that was filled via loadupTable
+  public void queryTable(int times) throws BlurException, TException {
+    long start = System.currentTimeMillis();
+    BlurQuery bq = new BlurQuery();
+    bq.fetch = 10;
+    for (int i = 1; i <= times; i++) {
+      SimpleQuery sq = new SimpleQuery();
+      sq.queryStr = "numberField:" + random.nextInt(1000);
+      sq.superQueryOn = true;
+      bq.simpleQuery = sq;
+      client.query(table, bq);
+      if (i % 1000 == 0) {
+        System.out.println("queries: " + i + " times " + df.format((i / ((System.currentTimeMillis() - start + 0.0) / 1000))) + " queries/s");
+      }
+    }
+    System.out.println("queries: " + times + " times " + df.format((times / ((System.currentTimeMillis() - start + 0.0) / 1000))) + " queries/s");
+
+  }
+
+  public void loadupTable(int rows) throws BlurException, TException, IOException {
+
+    long start = System.currentTimeMillis();
+
+    long buildTotal = 0;
+    RowMutation mutation = new RowMutation();
+
+    for (int i = 1; i <= rows; i++) {
+      long buildStart = System.currentTimeMillis();
+      mutation.clear();
+      mutation.table = table;
+      mutation.waitToBeVisible = false;
+      mutation.rowId = UUID.randomUUID().toString();
+      mutation.addToRecordMutations(newRecordMutation("test", "test-" + i, newColumn("uuidField", UUID.randomUUID().toString()), newColumn("numberField", i + ""),
+          newColumn("fatTextField", randomString(1000))));
+      mutation.rowMutationType = RowMutationType.REPLACE_ROW;
+
+      if (i % 50 == 0) {
+        System.out.println("loaded: " + i + " around " + df.format((i / ((System.currentTimeMillis() - start + 0.0) / 1000))) + " rows/s");
+        System.out.println("Total time: " + (System.currentTimeMillis() - start + 0.0) / 1000 + " Build time: " + ((buildTotal / 1000) + 0.0) + " " + buildTotal);
+      }
+
+      buildTotal += System.currentTimeMillis() - buildStart;
+
+      client.mutate(mutation);
+
+    }
+    System.out.println("loaded: " + rows + " around " + df.format((rows / ((System.currentTimeMillis() - start + 0.0) / 1000))) + " rows/s");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java
new file mode 100644
index 0000000..2b4e697
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java
@@ -0,0 +1,50 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class CreateTable {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    String connectionStr = args[0];
+    final String cluster = args[1];
+    final String tableName = args[2];
+    int shardCount = Integer.parseInt(args[3]);
+    String uri = args[4];
+
+    final TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.analyzerDefinition = new AnalyzerDefinition();
+    tableDescriptor.cluster = cluster;
+    tableDescriptor.name = tableName;
+    tableDescriptor.readOnly = false;
+
+    tableDescriptor.shardCount = shardCount;
+    tableDescriptor.tableUri = uri;
+
+    Iface client = BlurClient.getClient(connectionStr);
+    client.createTable(tableDescriptor);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/DisableTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/DisableTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/DisableTable.java
new file mode 100644
index 0000000..c86689a
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/DisableTable.java
@@ -0,0 +1,36 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class DisableTable {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    String connectionStr = args[0];
+    final String tableName = args[1];
+
+    Iface client = BlurClient.getClient(connectionStr);
+    client.disableTable(tableName);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/EnableTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/EnableTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/EnableTable.java
new file mode 100644
index 0000000..b4186a6
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/EnableTable.java
@@ -0,0 +1,36 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class EnableTable {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    String connectionStr = args[0];
+    final String tableName = args[1];
+
+    Iface client = BlurClient.getClient(connectionStr);
+    client.enableTable(tableName);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/ListTables.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/ListTables.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/ListTables.java
new file mode 100644
index 0000000..3405d42
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/ListTables.java
@@ -0,0 +1,35 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class ListTables {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    String connectionStr = args[0];
+
+    Iface client = BlurClient.getClient(connectionStr);
+    System.out.println(client.tableList());
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadData.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadData.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadData.java
new file mode 100644
index 0000000..7b71c88
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadData.java
@@ -0,0 +1,129 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class LoadData {
+
+  private static Random random = new Random();
+  private static List<String> words = new ArrayList<String>();
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    loadWords();
+    final boolean wal = true;
+    final int numberOfColumns = 3;
+    int numberRows = 100000;
+    final int numberRecordsPerRow = 3;
+    final int numberOfFamilies = 3;
+    final int numberOfWords = 30;
+    int count = 0;
+    int max = 100;
+    long start = System.currentTimeMillis();
+    final String table = "test_table";
+    for (int i = 0; i < numberRows; i++) {
+      if (count >= max) {
+        double seconds = (System.currentTimeMillis() - start) / 1000.0;
+        double rate = i / seconds;
+        System.out.println("Rows indexed [" + i + "] at [" + rate + "/s]");
+        count = 0;
+      }
+
+      Iface client = BlurClient.getClient(args[0]);
+      RowMutation mutation = new RowMutation();
+      mutation.setTable(table);
+      String rowId = getRowId();
+      mutation.setRowId(rowId);
+      mutation.setWal(wal);
+      mutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+      for (int j = 0; j < numberRecordsPerRow; j++) {
+        mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies, numberOfWords));
+      }
+      client.mutate(mutation);
+      count++;
+    }
+  }
+
+  private static void loadWords() throws IOException {
+    InputStream inputStream = LoadData.class.getResourceAsStream("words.txt");
+    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+    String word;
+    while ((word = reader.readLine()) != null) {
+      words.add(word.trim());
+    }
+    reader.close();
+  }
+
+  protected static RecordMutation getRecordMutation(int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    RecordMutation recordMutation = new RecordMutation();
+    recordMutation.setRecord(getRecord(numberOfColumns, numberOfFamilies, numberOfWords));
+    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+    return recordMutation;
+  }
+
+  private static Record getRecord(int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    Record record = new Record();
+    record.setRecordId(getRowId());
+    record.setFamily(getFamily(numberOfFamilies));
+    for (int i = 0; i < numberOfColumns; i++) {
+      record.addToColumns(new Column("col" + i, getWords(numberOfWords)));
+    }
+    return record;
+  }
+
+  private static String getWords(int numberOfWords) {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < numberOfWords; i++) {
+      if (i != 0) {
+        builder.append(' ');
+      }
+      builder.append(getWord());
+    }
+    return builder.toString();
+  }
+
+  private static String getFamily(int numberOfFamilies) {
+    return "fam" + random.nextInt(numberOfFamilies);
+  }
+
+  private static String getWord() {
+    return words.get(random.nextInt(words.size()));
+  }
+
+  protected static String getRowId() {
+    return Long.toString(Math.abs(random.nextLong())) + "-" + Long.toString(Math.abs(random.nextLong()));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataAsync.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataAsync.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataAsync.java
new file mode 100644
index 0000000..9962747
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataAsync.java
@@ -0,0 +1,146 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.blur.thrift.AsyncClientPool;
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+import org.apache.blur.thrift.generated.Blur.AsyncIface;
+import org.apache.blur.thrift.generated.Blur.AsyncClient.mutate_call;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+
+
+public class LoadDataAsync {
+
+  private static Random random = new Random();
+  private static List<String> words = new ArrayList<String>();
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    loadWords();
+    final boolean wal = false;
+    final int numberOfColumns = 3;
+    int numberRows = 100000;
+    final int numberRecordsPerRow = 2;
+    final int numberOfFamilies = 3;
+    final int numberOfWords = 30;
+    int count = 0;
+    int max = 100;
+    long start = System.currentTimeMillis();
+    final String table = "test-table";
+    AsyncClientPool pool = new AsyncClientPool();
+    AsyncIface client = pool.getClient(Blur.AsyncIface.class, args[0]);
+    for (int i = 0; i < numberRows; i++) {
+      if (count >= max) {
+        double seconds = (System.currentTimeMillis() - start) / 1000.0;
+        double rate = i / seconds;
+        System.out.println("Rows indexed [" + i + "] at [" + rate + "/s]");
+        count = 0;
+      }
+      client.mutate(getRowMutation(table, wal, numberRecordsPerRow, numberOfColumns, numberOfFamilies, numberOfWords), new AsyncMethodCallback<Blur.AsyncClient.mutate_call>() {
+        @Override
+        public void onError(Exception exception) {
+
+        }
+
+        @Override
+        public void onComplete(mutate_call response) {
+
+        }
+      });
+      count++;
+    }
+  }
+
+  private static RowMutation getRowMutation(String table, boolean wal, int numberRecordsPerRow, int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    RowMutation mutation = new RowMutation();
+    mutation.setTable(table);
+    String rowId = getRowId();
+    mutation.setRowId(rowId);
+    mutation.setWal(wal);
+    mutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+    for (int j = 0; j < numberRecordsPerRow; j++) {
+      mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies, numberOfWords));
+    }
+    return mutation;
+  }
+
+  private static void loadWords() throws IOException {
+    InputStream inputStream = LoadDataAsync.class.getResourceAsStream("words.txt");
+    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+    String word;
+    while ((word = reader.readLine()) != null) {
+      words.add(word.trim());
+    }
+    reader.close();
+  }
+
+  protected static RecordMutation getRecordMutation(int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    RecordMutation recordMutation = new RecordMutation();
+    recordMutation.setRecord(getRecord(numberOfColumns, numberOfFamilies, numberOfWords));
+    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+    return recordMutation;
+  }
+
+  private static Record getRecord(int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    Record record = new Record();
+    record.setRecordId(getRowId());
+    record.setFamily(getFamily(numberOfFamilies));
+    for (int i = 0; i < numberOfColumns; i++) {
+      record.addToColumns(new Column("col" + i, getWords(numberOfWords)));
+    }
+    return record;
+  }
+
+  private static String getWords(int numberOfWords) {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < numberOfWords; i++) {
+      if (i != 0) {
+        builder.append(' ');
+      }
+      builder.append(getWord());
+    }
+    return builder.toString();
+  }
+
+  private static String getFamily(int numberOfFamilies) {
+    return "fam" + random.nextInt(numberOfFamilies);
+  }
+
+  private static String getWord() {
+    return words.get(random.nextInt(words.size()));
+  }
+
+  protected static String getRowId() {
+    return Long.toString(Math.abs(random.nextLong())) + "-" + Long.toString(Math.abs(random.nextLong()));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataAsyncContinuously.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataAsyncContinuously.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataAsyncContinuously.java
new file mode 100644
index 0000000..b7caaec
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataAsyncContinuously.java
@@ -0,0 +1,154 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.blur.thrift.AsyncClientPool;
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+import org.apache.blur.thrift.generated.Blur.AsyncIface;
+import org.apache.blur.thrift.generated.Blur.AsyncClient.mutate_call;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+
+
+public class LoadDataAsyncContinuously {
+
+  private static Random random = new Random();
+  private static List<String> words = new ArrayList<String>();
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    loadWords();
+    while (true) {
+      final boolean wal = true;
+      final int numberOfColumns = 3;
+      int numberRows = 100000;
+      final int numberRecordsPerRow = 2;
+      final int numberOfFamilies = 3;
+      final int numberOfWords = 30;
+      int count = 0;
+      int max = 1000;
+      long start = System.currentTimeMillis();
+      final String table = "test1";
+      AsyncClientPool pool = new AsyncClientPool();
+      AsyncIface client = pool.getClient(Blur.AsyncIface.class, args[0]);
+      for (int i = 0; i < numberRows; i++) {
+        if (count >= max) {
+          double seconds = (System.currentTimeMillis() - start) / 1000.0;
+          double rate = i / seconds;
+          System.out.println("Rows indexed [" + i + "] at [" + rate + "/s]");
+          count = 0;
+        }
+        client.mutate(getRowMutation(i, table, wal, numberRecordsPerRow, numberOfColumns, numberOfFamilies, numberOfWords),
+            new AsyncMethodCallback<Blur.AsyncClient.mutate_call>() {
+              @Override
+              public void onError(Exception exception) {
+                exception.printStackTrace();
+              }
+
+              @Override
+              public void onComplete(mutate_call response) {
+                try {
+                  response.getResult();
+                } catch (BlurException e) {
+                  e.printStackTrace();
+                } catch (TException e) {
+                  e.printStackTrace();
+                }
+              }
+            });
+        count++;
+      }
+    }
+  }
+
+  private static RowMutation getRowMutation(int rowid, String table, boolean wal, int numberRecordsPerRow, int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    RowMutation mutation = new RowMutation();
+    mutation.setTable(table);
+    mutation.setRowId(Integer.toString(rowid));
+    mutation.setWal(wal);
+    mutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+    for (int j = 0; j < numberRecordsPerRow; j++) {
+      mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies, numberOfWords));
+    }
+    return mutation;
+  }
+
+  private static void loadWords() throws IOException {
+    InputStream inputStream = LoadDataAsyncContinuously.class.getResourceAsStream("words.txt");
+    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+    String word;
+    while ((word = reader.readLine()) != null) {
+      words.add(word.trim());
+    }
+    reader.close();
+  }
+
+  protected static RecordMutation getRecordMutation(int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    RecordMutation recordMutation = new RecordMutation();
+    recordMutation.setRecord(getRecord(numberOfColumns, numberOfFamilies, numberOfWords));
+    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+    return recordMutation;
+  }
+
+  private static Record getRecord(int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    Record record = new Record();
+    record.setRecordId(getRowId());
+    record.setFamily(getFamily(numberOfFamilies));
+    for (int i = 0; i < numberOfColumns; i++) {
+      record.addToColumns(new Column("col" + i, getWords(numberOfWords)));
+    }
+    return record;
+  }
+
+  private static String getWords(int numberOfWords) {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < numberOfWords; i++) {
+      if (i != 0) {
+        builder.append(' ');
+      }
+      builder.append(getWord());
+    }
+    return builder.toString();
+  }
+
+  private static String getFamily(int numberOfFamilies) {
+    return "fam" + random.nextInt(numberOfFamilies);
+  }
+
+  private static String getWord() {
+    return words.get(random.nextInt(words.size()));
+  }
+
+  protected static String getRowId() {
+    return Long.toString(Math.abs(random.nextLong())) + "-" + Long.toString(Math.abs(random.nextLong()));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataContinuously.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataContinuously.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataContinuously.java
new file mode 100644
index 0000000..9902bc9
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/LoadDataContinuously.java
@@ -0,0 +1,174 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class LoadDataContinuously {
+
+  private static Random random = new Random();
+  public static List<String> words = new ArrayList<String>();
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    if (!(args.length == 8 || args.length == 9)) {
+      System.err
+          .println(LoadDataContinuously.class.getName()
+              + " <host1:port1,host2:port2> <table name> <WAL true|false> <# of columns per record> <# of records per row> <# of column families> <# of words per record> <time in seconds between reporting progress> <*optional path to word dictionary>");
+      System.exit(1);
+    }
+    if (args.length == 9) {
+      loadWords(args[8]);
+    } else {
+      loadWords(null);
+    }
+
+    final Iface client = BlurClient.getClient(args[0]);
+    final String table = args[1];
+    final boolean wal = Boolean.parseBoolean(args[2]);
+    final int numberOfColumns = Integer.parseInt(args[3]);
+    final int numberRecordsPerRow = Integer.parseInt(args[4]);
+    final int numberOfFamilies = Integer.parseInt(args[5]);
+    final int numberOfWords = Integer.parseInt(args[6]);
+    final long timeBetweenReporting = TimeUnit.SECONDS.toMillis(Integer.parseInt(args[7]));
+    final long start = System.currentTimeMillis();
+
+    long s = start;
+    long recordCountTotal = 0;
+    long rowCount = 0;
+
+    int batchSize = 100;
+
+    List<RowMutation> batch = new ArrayList<RowMutation>();
+
+    long recordCount = 0;
+    long totalTime = 0;
+    long calls = 0;
+    while (true) {
+      long now = System.currentTimeMillis();
+      if (s + timeBetweenReporting < now) {
+        double avgSeconds = (now - start) / 1000.0;
+        double seconds = (now - s) / 1000.0;
+        double avgRate = recordCountTotal / avgSeconds;
+        double rate = recordCount / seconds;
+        double latency = (totalTime / 1000000.0) / calls;
+        System.out.println(System.currentTimeMillis() + "," + recordCountTotal + "," + rowCount + "," + latency + "," + rate + "," + avgRate);
+        s = now;
+        recordCount = 0;
+        totalTime = 0;
+        calls = 0;
+      }
+
+      RowMutation mutation = new RowMutation();
+      mutation.setTable(table);
+      String rowId = getRowId();
+      mutation.setRowId(rowId);
+      mutation.setWal(wal);
+      mutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+      for (int j = 0; j < numberRecordsPerRow; j++) {
+        mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies, numberOfWords));
+      }
+      batch.add(mutation);
+      if (batch.size() >= batchSize) {
+        long sm = System.nanoTime();
+        client.mutateBatch(batch);
+        long em = System.nanoTime();
+        calls++;
+        totalTime += (em - sm);
+        batch.clear();
+      }
+      rowCount++;
+      recordCount += numberRecordsPerRow;
+      recordCountTotal += numberRecordsPerRow;
+    }
+  }
+
+  public static void loadWords(String path) throws IOException {
+    InputStream inputStream;
+    if (path == null) {
+      inputStream = LoadDataContinuously.class.getResourceAsStream("words.txt");
+    } else {
+      inputStream = new FileInputStream(path);
+    }
+    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+    String word;
+    while ((word = reader.readLine()) != null) {
+      words.add(word.trim());
+    }
+    reader.close();
+  }
+
+  protected static RecordMutation getRecordMutation(int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    RecordMutation recordMutation = new RecordMutation();
+    recordMutation.setRecord(getRecord(numberOfColumns, numberOfFamilies, numberOfWords));
+    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+    return recordMutation;
+  }
+
+  private static Record getRecord(int numberOfColumns, int numberOfFamilies, int numberOfWords) {
+    Record record = new Record();
+    record.setRecordId(getRowId());
+    record.setFamily(getFamily(numberOfFamilies));
+    for (int i = 0; i < numberOfColumns; i++) {
+      record.addToColumns(new Column("col" + i, getWords(numberOfWords)));
+    }
+    return record;
+  }
+
+  private static String getWords(int numberOfWords) {
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < numberOfWords; i++) {
+      if (i != 0) {
+        builder.append(' ');
+      }
+      builder.append(getWord());
+    }
+    return builder.toString();
+  }
+
+  private static String getFamily(int numberOfFamilies) {
+    return "fam" + random.nextInt(numberOfFamilies);
+  }
+
+  private static String getWord() {
+    return words.get(random.nextInt(words.size()));
+  }
+
+  protected static String getRowId() {
+    return Long.toString(Math.abs(random.nextLong())) + "-" + Long.toString(Math.abs(random.nextLong()));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/OptimizeTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/OptimizeTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/OptimizeTable.java
new file mode 100644
index 0000000..1515ed9
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/OptimizeTable.java
@@ -0,0 +1,35 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class OptimizeTable {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    final String tableName = args[1];
+    final int segmentCount = Integer.parseInt(args[2]);
+    Iface client = BlurClient.getClient(args[0]);
+    client.optimize(tableName, segmentCount);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RandomSearchTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RandomSearchTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RandomSearchTable.java
new file mode 100644
index 0000000..860a44a
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RandomSearchTable.java
@@ -0,0 +1,120 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.Schema;
+import org.apache.blur.thrift.generated.SimpleQuery;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class RandomSearchTable {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    String connectionStr = args[0];
+    final String tableName = args[1];
+    int numberOfTerms = Integer.parseInt(args[2]);
+    int numberOfSearchesPerPass = Integer.parseInt(args[3]);
+    int numberOfTermsPerQuery = Integer.parseInt(args[4]);
+    List<String> sampleOfTerms = getSampleOfTerms(connectionStr, tableName, numberOfTerms);
+    // for (String term : sampleOfTerms) {
+    // System.out.println(term);
+    // }
+    runSearches(connectionStr, tableName, sampleOfTerms, numberOfSearchesPerPass, numberOfTermsPerQuery);
+  }
+
+  private static void runSearches(String connectionStr, final String tableName, List<String> sampleOfTerms, int numberOfSearchesPerPass, int numberOfTermsPerQuery)
+      throws BlurException, TException, IOException {
+    Random random = new Random();
+    StringBuilder builder = new StringBuilder();
+    for (int i = 0; i < numberOfSearchesPerPass; i++) {
+
+      builder.setLength(0);
+      String query = generateQuery(builder, random, sampleOfTerms, numberOfTermsPerQuery);
+      System.out.println(query);
+      final BlurQuery blurQuery = new BlurQuery();
+      blurQuery.simpleQuery = new SimpleQuery();
+      blurQuery.simpleQuery.queryStr = query;
+      long start = System.nanoTime();
+
+      Iface client = BlurClient.getClient(connectionStr);
+      BlurResults results = client.query(tableName, blurQuery);
+      long end = System.nanoTime();
+      System.out.println((end - start) / 1000000.0 + " ms " + results.totalResults);
+    }
+  }
+
+  private static String generateQuery(StringBuilder builder, Random random, List<String> sampleOfTerms, int numberOfTermsPerQuery) {
+    for (int i = 0; i < numberOfTermsPerQuery; i++) {
+      builder.append(getRandomTerm(sampleOfTerms, random)).append(' ');
+    }
+    return builder.toString().trim();
+  }
+
+  private static String getRandomTerm(List<String> sampleOfTerms, Random random) {
+    int index = random.nextInt(sampleOfTerms.size());
+    return sampleOfTerms.get(index);
+  }
+
+  private static List<String> getSampleOfTerms(String connectionStr, String tableName, int numberOfTerms) throws BlurException, TException, IOException {
+    List<String> sampleOfTerms = new ArrayList<String>();
+    Set<String> fields = getFields(connectionStr, tableName);
+    for (String field : fields) {
+      Set<String> randomSampleOfTerms = getRandomSampleOfTerms(connectionStr, tableName, field, numberOfTerms);
+      for (String term : randomSampleOfTerms) {
+        sampleOfTerms.add(field + ":" + term);
+      }
+    }
+    Collections.shuffle(sampleOfTerms);
+    return sampleOfTerms;
+  }
+
+  private static Set<String> getRandomSampleOfTerms(String connectionStr, final String tableName, final String field, final int numberOfTerms) throws BlurException, TException,
+      IOException {
+    Iface client = BlurClient.getClient(connectionStr);
+    String[] split = field.split("\\.");
+    String columnFamily = split[0];
+    String columnName = split[1];
+    List<String> terms = client.terms(tableName, columnFamily, columnName, "", (short) numberOfTerms);
+    return new HashSet<String>(terms);
+  }
+
+  private static Set<String> getFields(String connectionStr, final String tableName) throws BlurException, TException, IOException {
+    Iface client = BlurClient.getClient(connectionStr);
+    Schema schema = client.schema(tableName);
+    Set<String> fields = new HashSet<String>();
+    for (String cf : schema.columnFamilies.keySet()) {
+      for (String field : schema.columnFamilies.get(cf)) {
+        fields.add(cf + "." + field);
+      }
+    }
+    return fields;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RandomSearchTableContinuously.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RandomSearchTableContinuously.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RandomSearchTableContinuously.java
new file mode 100644
index 0000000..8d2d72e
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RandomSearchTableContinuously.java
@@ -0,0 +1,161 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.Schema;
+import org.apache.blur.thrift.generated.Selector;
+import org.apache.blur.thrift.generated.SimpleQuery;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class RandomSearchTableContinuously {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    if (args.length != 6) {
+      System.err
+          .println(RandomSearchTableContinuously.class.getName()
+              + " <host1:port1,host2:port2> <table name> <# of terms to load into memory per pass> <# of searches per pass> <# of terms per query> <time in seconds between reporting progress>");
+      System.exit(1);
+    }
+    final String connectionStr = args[0];
+    final String tableName = args[1];
+    final int numberOfTerms = Integer.parseInt(args[2]);
+    final int numberOfSearchesPerPass = Integer.parseInt(args[3]);
+    final int numberOfTermsPerQuery = Integer.parseInt(args[4]);
+    final long timeBetweenReporting = TimeUnit.SECONDS.toMillis(Integer.parseInt(args[5]));
+    List<String> sampleOfTerms = getSampleOfTerms(connectionStr, tableName, numberOfTerms);
+    while (true) {
+      runSearches(connectionStr, tableName, sampleOfTerms, numberOfSearchesPerPass, numberOfTermsPerQuery, timeBetweenReporting);
+    }
+  }
+
+  private static void runSearches(String connectionStr, final String tableName, List<String> sampleOfTerms, int numberOfSearchesPerPass, int numberOfTermsPerQuery,
+      long timeBetweenReporting) throws BlurException, TException, IOException {
+    Random random = new Random();
+    StringBuilder builder = new StringBuilder();
+    final long start = System.currentTimeMillis();
+    long s = start;
+    long responseTime = 0;
+    int count = 0;
+    long resultCount = 0;
+    Iface client = BlurClient.getClient(connectionStr);
+    int i;
+    for (i = 0; i < numberOfSearchesPerPass; i++) {
+      long now = System.currentTimeMillis();
+      if (s + timeBetweenReporting < now) {
+        double avgSeconds = (now - start) / 1000.0;
+        double seconds = (now - s) / 1000.0;
+        double avgRate = i / avgSeconds;
+        double rate = count / seconds;
+        double responseTimeAvg = (responseTime / (double) count) / 1000000.0;
+        System.out.println(System.currentTimeMillis() + "," + i + "," + responseTimeAvg + "," + rate + "," + avgRate + "," + resultCount + "," + getCount(client, tableName));
+        s = now;
+        responseTime = 0;
+        count = 0;
+        resultCount = 0;
+      }
+
+      builder.setLength(0);
+      String query = generateQuery(builder, random, sampleOfTerms, numberOfTermsPerQuery);
+      final BlurQuery blurQuery = new BlurQuery();
+      blurQuery.simpleQuery = new SimpleQuery();
+      blurQuery.simpleQuery.queryStr = query;
+      blurQuery.cacheResult = false;
+      blurQuery.selector = new Selector();
+      long qs = System.nanoTime();
+
+      BlurResults results = client.query(tableName, blurQuery);
+      long qe = System.nanoTime();
+      resultCount += results.totalResults;
+      responseTime += (qe - qs);
+      count++;
+    }
+  }
+
+  private static long getCount(Iface client, String tableName) throws BlurException, TException {
+    BlurQuery bq = new BlurQuery();
+    bq.simpleQuery = new SimpleQuery();
+    bq.simpleQuery.queryStr = "*";
+    bq.simpleQuery.superQueryOn = false;
+    bq.cacheResult = false;
+    bq.useCacheIfPresent = false;
+    BlurResults results = client.query(tableName, bq);
+    return results.totalResults;
+  }
+
+  private static String generateQuery(StringBuilder builder, Random random, List<String> sampleOfTerms, int numberOfTermsPerQuery) {
+    for (int i = 0; i < numberOfTermsPerQuery; i++) {
+      builder.append(getRandomTerm(sampleOfTerms, random)).append(' ');
+    }
+    return builder.toString().trim();
+  }
+
+  private static String getRandomTerm(List<String> sampleOfTerms, Random random) {
+    int index = random.nextInt(sampleOfTerms.size());
+    return sampleOfTerms.get(index);
+  }
+
+  private static List<String> getSampleOfTerms(String connectionStr, String tableName, int numberOfTerms) throws BlurException, TException, IOException {
+    List<String> sampleOfTerms = new ArrayList<String>();
+    Set<String> fields = getFields(connectionStr, tableName);
+    for (String field : fields) {
+      Set<String> randomSampleOfTerms = getRandomSampleOfTerms(connectionStr, tableName, field, numberOfTerms);
+      for (String term : randomSampleOfTerms) {
+        sampleOfTerms.add(field + ":" + term);
+      }
+    }
+    Collections.shuffle(sampleOfTerms);
+    return sampleOfTerms;
+  }
+
+  private static Set<String> getRandomSampleOfTerms(String connectionStr, final String tableName, final String field, final int numberOfTerms) throws BlurException, TException,
+      IOException {
+    Iface client = BlurClient.getClient(connectionStr);
+    String[] split = field.split("\\.");
+    String columnFamily = split[0];
+    String columnName = split[1];
+    List<String> terms = client.terms(tableName, columnFamily, columnName, "", (short) numberOfTerms);
+    return new HashSet<String>(terms);
+  }
+
+  private static Set<String> getFields(String connectionStr, final String tableName) throws BlurException, TException, IOException {
+    Iface client = BlurClient.getClient(connectionStr);
+    Schema schema = client.schema(tableName);
+    Set<String> fields = new HashSet<String>();
+    for (String cf : schema.columnFamilies.keySet()) {
+      for (String field : schema.columnFamilies.get(cf)) {
+        fields.add(cf + "." + field);
+      }
+    }
+    return fields;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RapidlyCreateAndDeleteTables.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RapidlyCreateAndDeleteTables.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RapidlyCreateAndDeleteTables.java
new file mode 100644
index 0000000..96713f4
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RapidlyCreateAndDeleteTables.java
@@ -0,0 +1,93 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.apache.blur.utils.BlurUtil.newColumn;
+import static org.apache.blur.utils.BlurUtil.newRecordMutation;
+
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class RapidlyCreateAndDeleteTables {
+
+  public static void main(String[] args) throws BlurException, TException {
+    String connectionStr = args[0];
+    final String cluster = args[1];
+    String uri = args[2];
+    int shardCount = 1;
+    Iface client = BlurClient.getClient(connectionStr);
+    while (true) {
+      String tableName = UUID.randomUUID().toString();
+      System.out.println("Creating [" + tableName + "]");
+      boolean readOnly = createTable(client, cluster, uri, shardCount, tableName);
+      if (!readOnly) {
+        System.out.println("Loading [" + tableName + "]");
+        loadTable(client, tableName);
+      }
+      System.out.println("Disabling [" + tableName + "]");
+      disable(client, tableName);
+      System.out.println("Removing [" + tableName + "]");
+      delete(client, tableName);
+    }
+  }
+
+  private static void disable(Iface client, String tableName) throws BlurException, TException {
+    client.disableTable(tableName);
+  }
+
+  private static void delete(Iface client, String tableName) throws BlurException, TException {
+    client.removeTable(tableName, true);
+  }
+
+  private static void loadTable(Iface client, String tableName) throws BlurException, TException {
+    RowMutation mutation = new RowMutation();
+    mutation.table = tableName;
+    mutation.waitToBeVisible = true;
+    mutation.rowId = "test";
+    mutation.addToRecordMutations(newRecordMutation("test", "test", newColumn("test", "test")));
+    mutation.rowMutationType = RowMutationType.REPLACE_ROW;
+    client.mutate(mutation);
+  }
+
+  private static boolean createTable(Iface client, final String cluster, String uri, int shardCount, String tableName) throws BlurException, TException {
+    Random random = new Random();
+    final TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.analyzerDefinition = new AnalyzerDefinition();
+    tableDescriptor.cluster = cluster;
+
+    tableDescriptor.name = tableName;
+    tableDescriptor.readOnly = random.nextBoolean();
+
+    tableDescriptor.shardCount = shardCount;
+    tableDescriptor.tableUri = uri + "/" + tableName;
+
+    client.createTable(tableDescriptor);
+
+    return tableDescriptor.readOnly;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RemoveTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RemoveTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RemoveTable.java
new file mode 100644
index 0000000..4089eb7
--- /dev/null
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/RemoveTable.java
@@ -0,0 +1,36 @@
+package org.apache.blur.testsuite;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.thrift.TException;
+
+
+public class RemoveTable {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    String connectionStr = args[0];
+    final String tableName = args[1];
+
+    Iface client = BlurClient.getClient(connectionStr);
+    client.removeTable(tableName, true);
+  }
+}


Mime
View raw message