incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [14/51] [abbrv] git commit: Added a test for in bulk performance.
Date Tue, 11 Dec 2012 02:20:58 GMT
Added a test for in bulk performance.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/68cf6ed8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/68cf6ed8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/68cf6ed8

Branch: refs/heads/0.2-dev
Commit: 68cf6ed8ff5153323b1421571c4be91c90c980af
Parents: ce72e54
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Nov 24 21:47:06 2012 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Nov 24 21:47:06 2012 -0500

----------------------------------------------------------------------
 .../org/apache/blur/thrift/ITBlurClusterTest.java  |   55 +++++++++++++++
 1 files changed, 55 insertions(+), 0 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/68cf6ed8/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
index 83d9ceb..28d1144 100644
--- a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
@@ -190,4 +190,59 @@ public class ITBlurClusterTest {
     }
     assertEquals(SHARD_COUNT * 100, totalHits);
   }
+  
+  @Test
+  public void testAddDocumentsPerformance() throws BlurException, TException, InterruptedException,
IOException {
+    Iface client = getClient();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName("test-add-table-perf");
+    tableDescriptor.setShardCount(SHARD_COUNT);
+    tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test-add-table-perf");
+    client.createTable(tableDescriptor);
+
+    List<Generation> generations = new ArrayList<Generation>();
+    int length = 100000;
+    int batch = 100;
+    long t1 = System.nanoTime();
+    for (int s = 0; s < SHARD_COUNT; s++) {
+      List<Document> documents = new ArrayList<Document>();
+      for (int i = 0; i < length; i++) {
+        Document doc = new Document();
+        doc.addToFields(new Field("id", ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()),
TYPE.STRING, 1.0));
+        doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING,
1.0));
+        documents.add(doc);
+        if (documents.size() >= batch) {
+          MutateOptions options = new MutateOptions();
+          options.setTable(tableDescriptor.getName());
+          options.setShardIndex(s);
+          generations.addAll(client.addDocuments(options, documents));
+          documents.clear();
+        }
+      }
+      MutateOptions options = new MutateOptions();
+      options.setTable(tableDescriptor.getName());
+      options.setShardIndex(s);
+      generations.addAll(client.addDocuments(options, documents));
+    }
+    long t2 = System.nanoTime();
+
+    client.blockUntilGenerationIsVisible(generations, true);
+    
+    long t3 = System.nanoTime();
+    
+    System.out.println("Add time [" + (t2 - t1) / 1000000000.0 + "]");
+    System.out.println("Block time [" + (t3 - t2) / 1000000000.0 + "]");
+
+    Session session = client.openReadSession(tableDescriptor.getName());
+    QueryArgs queryArgs = new QueryArgs();
+    Term term = new Term("value", "test");
+    TermQuery query = new TermQuery(term);
+    queryArgs.setQuery(ThriftLuceneConversion.toBytes(query));
+    List<TopFieldDocs> results = client.search(session, queryArgs);
+    long totalHits = 0;
+    for (TopFieldDocs fieldDocs : results) {
+      totalHits += fieldDocs.getTotalHits();
+    }
+    assertEquals(SHARD_COUNT * length, totalHits);
+  }
 }


Mime
View raw message