incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [18/47] git commit: cleaned up test class
Date Mon, 03 Sep 2012 00:32:19 GMT
cleaned up test class


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/71ab7cd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/71ab7cd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/71ab7cd5

Branch: refs/heads/master
Commit: 71ab7cd5a207d47f60ee812e16a0c8f59832d9ef
Parents: 3e7ea7f
Author: gbarton <garrett.barton@gmail.com>
Authored: Sun Aug 26 11:19:20 2012 -0400
Committer: gbarton <garrett.barton@gmail.com>
Committed: Sun Aug 26 11:19:20 2012 -0400

----------------------------------------------------------------------
 .../blur/testsuite/CreateInsertQueryRepeating.java |  237 +++++++--------
 1 files changed, 106 insertions(+), 131 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/71ab7cd5/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java
b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java
index 70824d1..96aea40 100644
--- a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java
+++ b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java
@@ -8,20 +8,13 @@ import java.util.Random;
 import java.util.UUID;
 
 import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
 
-import com.nearinfinity.blur.thrift.AsyncClientPool;
 import com.nearinfinity.blur.thrift.BlurClient;
 import com.nearinfinity.blur.thrift.generated.AnalyzerDefinition;
-import com.nearinfinity.blur.thrift.generated.Blur;
-import com.nearinfinity.blur.thrift.generated.Blur.AsyncClient.mutateBatch_call;
-import com.nearinfinity.blur.thrift.generated.Blur.AsyncClient.mutate_call;
-import com.nearinfinity.blur.thrift.generated.Blur.AsyncIface;
 import com.nearinfinity.blur.thrift.generated.Blur.Iface;
 import com.nearinfinity.blur.thrift.generated.BlurException;
 import com.nearinfinity.blur.thrift.generated.BlurQuery;
-import com.nearinfinity.blur.thrift.generated.ColumnDefinition;
-import com.nearinfinity.blur.thrift.generated.ColumnFamilyDefinition;
+import com.nearinfinity.blur.thrift.generated.BlurResults;
 import com.nearinfinity.blur.thrift.generated.RowMutation;
 import com.nearinfinity.blur.thrift.generated.RowMutationType;
 import com.nearinfinity.blur.thrift.generated.SimpleQuery;
@@ -29,9 +22,15 @@ import com.nearinfinity.blur.thrift.generated.TableDescriptor;
 
 import static com.nearinfinity.blur.utils.BlurUtil.*;
 
+/**
+ * Tests alot of things, mainly connecting to a blur cluster and slamming a bunch
+ * of rows in before querying for them.  I like to use it as a load test.
+ * @author gman
+ *
+ */
 public class CreateInsertQueryRepeating {
 
-	private static DecimalFormat df = new DecimalFormat("#,###,000.00");
+	private DecimalFormat df = new DecimalFormat("#,###,000.00");
 	private static final char[] symbols = new char[36];
 
 	static {
@@ -41,9 +40,37 @@ public class CreateInsertQueryRepeating {
 			symbols[idx] = (char) ('a' + idx - 10);
 	}
 
-	private static final Random random = new Random();
+	private String table = "test1";
+	private String host = "localhost";
+	private Iface client = null;
+	
+	public CreateInsertQueryRepeating(String host, String table) throws BlurException, TException,
IOException {
+		this.host = host;
+		this.table = table;
+		
+		//init
+		String connectionStr = host + ":40010";
+		String cluster = "default";
+		client = BlurClient.getClient(connectionStr);
+		
+		List<String> clusterList = client.shardClusterList();
+		if(clusterList != null && clusterList.size() > 0)
+			cluster = clusterList.get(0);
+		else
+			throw new IOException("cannot find a cluster to use :(");
+		
+		System.out.println("using cluster: " + cluster);
+
+		List<String> tableList = client.tableList();
+		if (tableList == null || !tableList.contains(table))
+			createTable(client, table, cluster);
+		else
+			System.out.println("table existed, did not create.");
+	}
+	
+	private final Random random = new Random();
 
-	public static String randomString(int length) {
+	public String randomString(int length) {
 		char[] buf = new char[length];
 
 		for (int idx = 0; idx < buf.length; ++idx)
@@ -51,7 +78,7 @@ public class CreateInsertQueryRepeating {
 		return new String(buf);
 	}
 
-	public static void getClusters(Iface client) {
+	public void getClusters(Iface client) {
 		try {
 			List<String> shardClusterList = client.shardClusterList();
 			for (String cluster : shardClusterList)
@@ -63,28 +90,22 @@ public class CreateInsertQueryRepeating {
 		}
 	}
 
-	public static void createTable(Iface client, String tableName,
-			String cluster) throws BlurException, TException {
+	public void createTable(Iface client, String tableName, String cluster) throws BlurException,
TException {
 		TableDescriptor td = new TableDescriptor();
 		td.analyzerDefinition = new AnalyzerDefinition();
-		ColumnFamilyDefinition cfd = new ColumnFamilyDefinition();
-		ColumnDefinition c = new ColumnDefinition(
-				"org.apache.lucene.analysis.standard.StandardAnalyzer", true,
-				null);
-		cfd.putToColumnDefinitions("cf1", c);
-		td.analyzerDefinition.putToColumnFamilyDefinitions("cf1", cfd);
 
 		td.name = tableName;
 		// TODO: doc doesnt say required, yet it barfs without it?
 		td.cluster = cluster == null ? "default" : cluster;
+		//auto enable table
 		td.isEnabled = true;
 
-		td.shardCount = 2;
+		//1 shard per server :)
+		td.shardCount = client.shardServerList(cluster).size();
 		td.readOnly = false;
-		// TODO: hardcodes bad
-		td.tableUri = "hdfs://localhost:8020/" + tableName;
+		// TODO: hardcodes bad, assuming NN on same node as BC
+		td.tableUri = "hdfs://" + host + ":8020/" + tableName;
 		client.createTable(td);
-		// client.enableTable(tableName);
 		System.out.println("table created");
 	}
 
@@ -95,44 +116,69 @@ public class CreateInsertQueryRepeating {
 	 * @throws IOException 
 	 */
 	public static void main(String[] args) throws BlurException, TException, IOException {
-		String connectionStr = "localhost:40010";
-		final String cluster = "default";
-		String uri = "hdfs://localhost:8020/test1";
-		int shardCount = 1;
-		Iface client = BlurClient.getClient(connectionStr);
-
-		List<String> tableList = client.tableList();
-		if (tableList == null || !tableList.contains("test1"))
-			createTable(client, "test1", cluster);
+		String host = "localhost";
+		String table = "test1";
+		
+		if(args != null) {
+			if(args.length >= 1)
+				host = args[0];
+			if(args.length == 2)
+				table = args[1];
+		}
+		
+		CreateInsertQueryRepeating test = new CreateInsertQueryRepeating(host, table);
 
+//		System.out.println("Testing joins real quick");
+//		test.testJoin();
+//		System.out.println("test done");
+		
 		System.out.println("Starting load");
-		loadupTable(client, "test1", 10);
+		test.loadupTable(100);
 		System.out.println("Finshed load");
 
 		System.out.println("query time!");
-		queryTable(client, "test1", 10000);
+		test.queryTable(50000);
 		System.out.println("query done!");
 
-		System.out.println("going into auto create test: ");
-
 		System.exit(0);
-		int i = 0;
-		while (i++ < 50) {
-			String tableName = UUID.randomUUID().toString();
-			System.out.println("Creating [" + tableName + "]");
-			createTable(client, cluster, uri, shardCount, tableName);
-			System.out.println("Loading [" + tableName + "]");
-			loadTable(client, tableName);
-			System.out.println("Disabling [" + tableName + "]");
-			disable(client, tableName);
-			System.out.println("Removing [" + tableName + "]");
-			delete(client, tableName);
-		}
 	}
 
+	@SuppressWarnings("unused")
+	private void testJoin() throws BlurException, TException {
+		RowMutation mutation = new RowMutation();
+		mutation.table = table;
+		mutation.waitToBeVisible = true;
+		mutation.rowId = "row1";
+		mutation.addToRecordMutations(newRecordMutation("cf1",
+				"recordid1", newColumn("col1","value1")));
+		mutation.addToRecordMutations(newRecordMutation("cf1",
+				"recordid2", newColumn("col2","value2")));
+		mutation.rowMutationType = RowMutationType.REPLACE_ROW;
+		client.mutate(mutation);
+		
+		List<String> joinTest = new ArrayList<String>();
+		joinTest.add("+cf1.col1:value1");
+		joinTest.add("+cf1.col2:value2");
+		joinTest.add("+cf1.col1:value1 +cf1.col2:value2");
+		joinTest.add("+(+cf1.col1:value1 nocf.nofield:somevalue) +(+cf1.col2.value2 nocf.nofield:somevalue)");
+		joinTest.add("+(+cf1.col1:value1) +(cf1.bla:bla +cf1.col2.value2)");
+		
+		for(String q : joinTest)
+			System.out.println(q + " hits: " + hits(client,table, q, true));
+	}
+	
+	private static long hits(Iface client, String table, String queryStr, boolean superQuery)
throws BlurException, TException {
+		BlurQuery bq = new BlurQuery();
+		SimpleQuery sq = new SimpleQuery();
+		sq.queryStr = queryStr;
+		sq.superQueryOn = superQuery;
+		bq.simpleQuery = sq;
+		BlurResults query = client.query(table, bq);
+		return query.totalResults;
+	}
+	
 	// really only useful against the table that was filled via loadupTable
-	private static void queryTable(Iface client, String tableName, int times)
-			throws BlurException, TException {
+	public void queryTable(int times) throws BlurException, TException {
 		long start = System.currentTimeMillis();
 		BlurQuery bq = new BlurQuery();
 		bq.fetch = 10;
@@ -141,7 +187,7 @@ public class CreateInsertQueryRepeating {
 			sq.queryStr = "numberField:" + random.nextInt(1000);
 			sq.superQueryOn = true;
 			bq.simpleQuery = sq;
-			client.query(tableName, bq);
+			client.query(table, bq);
 			if (i % 1000 == 0) {
 				System.out
 						.println("queries: "
@@ -160,23 +206,18 @@ public class CreateInsertQueryRepeating {
 
 	}
 
-	private static void loadupTable(Iface client, String tableName, int rows)
-			throws BlurException, TException, IOException {
-		AsyncClientPool pool = new AsyncClientPool();//10, 30000);
-		AsyncIface poolClient = pool.getClient(Blur.AsyncIface.class, "localhost:40010");
-		
+	public void loadupTable(int rows) throws BlurException, TException, IOException {
 		
 		long start = System.currentTimeMillis();
 		
-		List<RowMutation> mutates = new ArrayList<RowMutation>();
-		
 		long buildTotal = 0;
+		RowMutation mutation = new RowMutation();
 		
 		for (int i = 1; i <= rows; i++) {
 			long buildStart = System.currentTimeMillis();
-			RowMutation mutation = new RowMutation();
-			mutation.table = tableName;
-			mutation.waitToBeVisible = true;
+			mutation.clear();
+			mutation.table = table;
+			mutation.waitToBeVisible = false;
 			mutation.rowId = UUID.randomUUID().toString();
 			mutation.addToRecordMutations(newRecordMutation("test",
 					"test-" + i,
@@ -184,7 +225,7 @@ public class CreateInsertQueryRepeating {
 					newColumn("numberField", i + ""),
 					newColumn("fatTextField",randomString(1000))));
 			mutation.rowMutationType = RowMutationType.REPLACE_ROW;
-			
+
 			if (i % 50 == 0) {
 				System.out.println("loaded: " + i + " around "
 						+ df.format((i / ((System.currentTimeMillis() - start+0.0)/1000) ))
@@ -192,79 +233,13 @@ public class CreateInsertQueryRepeating {
 				System.out.println("Total time: " + (System.currentTimeMillis()-start+0.0)/1000 +
 						" Build time: "  + ((buildTotal/1000)+0.0) + " " + buildTotal);
 			}			
-			
-			buildTotal += System.currentTimeMillis() - buildStart;
-//			mutates.add(mutation);
-
-			poolClient.mutate(mutation, 
-					new AsyncMethodCallback<Blur.AsyncClient.mutate_call>() {
 
-						@Override
-						public void onComplete(mutate_call response) {
-						}
+			buildTotal += System.currentTimeMillis() - buildStart;
 
-						@Override
-						public void onError(Exception exception) {
-							exception.printStackTrace();
-						}
-					});
+			client.mutate(mutation);
 
-//			if(mutates.size() == 10) {
-//				pool.getClient(Blur.AsyncIface.class, "localhost:40010").mutateBatch(mutates,
-//						new AsyncMethodCallback<Blur.AsyncClient.mutateBatch_call>() {
-//						
-//						@Override
-//						public void onError(Exception exception) {	
-//							exception.printStackTrace();
-//						}
-//						
-//						@Override
-//						public void onComplete(mutateBatch_call response) {
-//						}
-//					});
-//				mutates.clear();
-//			}
 		}
 		System.out.println("loaded: " + rows + " around "
 				+ df.format((rows / ((System.currentTimeMillis() - start+0.0)/1000))) + " rows/s");
 	}
-
-	private static void disable(Iface client, String tableName)
-			throws BlurException, TException {
-		client.disableTable(tableName);
-	}
-
-	private static void delete(Iface client, String tableName)
-			throws BlurException, TException {
-		client.removeTable(tableName, true);
-	}
-
-	private static void loadTable(Iface client, String tableName)
-			throws BlurException, TException {
-		RowMutation mutation = new RowMutation();
-		mutation.table = tableName;
-		mutation.waitToBeVisible = true;
-		mutation.rowId = "test";
-		mutation.addToRecordMutations(newRecordMutation("test", "test",
-				newColumn("test", "test")));
-		mutation.rowMutationType = RowMutationType.REPLACE_ROW;
-		client.mutate(mutation);
-	}
-
-	private static void createTable(Iface client, final String cluster,
-			String uri, int shardCount, String tableName) throws BlurException,
-			TException {
-		final TableDescriptor tableDescriptor = new TableDescriptor();
-		tableDescriptor.analyzerDefinition = new AnalyzerDefinition();
-		tableDescriptor.cluster = cluster;
-
-		tableDescriptor.name = tableName;
-		tableDescriptor.readOnly = false;
-
-		tableDescriptor.shardCount = shardCount;
-		tableDescriptor.tableUri = uri + "/" + tableName;
-
-		client.createTable(tableDescriptor);
-	}
-
 }


Mime
View raw message