incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From twilli...@apache.org
Subject git commit: An initial implementation of a SolrJ SolrServer for Blur
Date Sat, 01 Mar 2014 03:43:29 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/apache-blur-0.2 e1c242b85 -> b6cdb2cce


An initial implementation of a SolrJ SolrServer for Blur


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b6cdb2cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b6cdb2cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b6cdb2cc

Branch: refs/heads/apache-blur-0.2
Commit: b6cdb2cce4ada876b92393e62b5174ae8e5b2a52
Parents: e1c242b
Author: twilliams <twilliams@apache.org>
Authored: Fri Feb 28 22:43:13 2014 -0500
Committer: twilliams <twilliams@apache.org>
Committed: Fri Feb 28 22:43:13 2014 -0500

----------------------------------------------------------------------
 whiteboard/slur/README.txt                      |  39 ++++
 whiteboard/slur/pom.xml                         |  64 +++++
 .../org/apache/blur/slur/RowMutationHelper.java | 114 +++++++++
 .../apache/blur/slur/SolrLookingBlurServer.java | 231 +++++++++++++++++++
 .../apache/blur/slur/RowMutationHelperTest.java | 116 ++++++++++
 .../blur/slur/SolrLookingBlurServerTest.java    | 197 ++++++++++++++++
 6 files changed, 761 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/README.txt
----------------------------------------------------------------------
diff --git a/whiteboard/slur/README.txt b/whiteboard/slur/README.txt
new file mode 100644
index 0000000..8b93c1d
--- /dev/null
+++ b/whiteboard/slur/README.txt
@@ -0,0 +1,39 @@
+== Overview ==
+This project explores using SolrJ to work with a Blur installation.
+
+== Mismatches ==
+o) Commits - Blur commits on an update; Solr expects and explicit - rather 
+	 than buffering, I've chosen to go ahead and follow Blur's model.
+o) Row/SolrInputDocument - The parent document in Solr seems to be a "real" 
+	 document, with fields. So far, you can either have subdocuments or fields
+	 on the main document, but not both.
+o) Optimize - Solr offers waitFlush, waitSearcher and Blur just offers maxSegments.
+	 
+== Usage ==
+	
+	= Create a server = 
+    SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(),
table);
+    
+  = Add a document =
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField("id", "1");
+    doc.addField("fam.value", "123");
+
+    server.add(doc);
+    
+  = Or add in batches =
+    List<SolrInputDocument> docs = Lists.newArrayList();
+
+    for (int i = 0; i < 100; i++) {
+      SolrInputDocument doc = new SolrInputDocument();
+      doc.addField("id", i);
+      doc.addField("fam.value", "12" + i);
+      docs.add(doc);
+    }
+
+    server.add(docs);
+    
+    
+    == Notes ==
+    **caveate is that I don't have experience with SolrJ, so this may very
+    well be dangerous.

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/pom.xml
----------------------------------------------------------------------
diff --git a/whiteboard/slur/pom.xml b/whiteboard/slur/pom.xml
new file mode 100644
index 0000000..3a03daa
--- /dev/null
+++ b/whiteboard/slur/pom.xml
@@ -0,0 +1,64 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>org.apache.blur</groupId>
+	<artifactId>Slur</artifactId>
+	<version>0.0.1-SNAPSHOT</version>
+	<name>Slur</name>
+	<description>A SolrJ implementation for Blur.</description>
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.solr</groupId>
+			<artifactId>solr-solrj</artifactId>
+			<version>4.6.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.blur</groupId>
+			<artifactId>blur-core</artifactId>
+			<version>0.2.2-incubating-SNAPSHOT</version>
+			<scope>runtime</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.blur</groupId>
+			<artifactId>blur-util</artifactId>
+			<version>0.2.2-incubating-SNAPSHOT</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.blur</groupId>
+			<artifactId>blur-thrift</artifactId>
+			<version>0.2.2-incubating-SNAPSHOT</version>
+		</dependency>
+		<dependency>
+			<groupId>commons-logging</groupId>
+			<artifactId>commons-logging</artifactId>
+			<version>1.1.1</version>
+		</dependency>
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava-collections</artifactId>
+			<version>r03</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.blur</groupId>
+			<artifactId>blur-core</artifactId>
+			<version>0.2.2-incubating-SNAPSHOT</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.blur</groupId>
+			<artifactId>blur-util</artifactId>
+			<version>0.2.2-incubating-SNAPSHOT</version>
+			<type>test-jar</type>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-test</artifactId>
+			<version>1.2.1</version>
+			<scope>test</scope>
+		</dependency>
+		
+		
+	</dependencies>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/src/main/java/org/apache/blur/slur/RowMutationHelper.java
----------------------------------------------------------------------
diff --git a/whiteboard/slur/src/main/java/org/apache/blur/slur/RowMutationHelper.java b/whiteboard/slur/src/main/java/org/apache/blur/slur/RowMutationHelper.java
new file mode 100644
index 0000000..80549e6
--- /dev/null
+++ b/whiteboard/slur/src/main/java/org/apache/blur/slur/RowMutationHelper.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.slur;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+
+import com.google.common.collect.Lists;
+
+public class RowMutationHelper {
+
+  public static List<RowMutation> from(Collection<SolrInputDocument> docs, String
table) {
+    List<RowMutation> mutations = Lists.newArrayList();
+    for(SolrInputDocument d: docs) {
+      System.out.println("DOC: " + d.getFieldValue("id"));
+      mutations.add(from(d, table));
+    }
+    return mutations;
+  }
+  
+  public static RowMutation from(SolrInputDocument doc, String table) {
+    validate(doc);
+
+    RowMutation mutate = new RowMutation();
+    mutate.setRowId(extractRowId(doc));
+    mutate.setTable(table);
+    List<RecordMutation> recordMutations = Lists.newArrayList();
+    if (doc.hasChildDocuments()) {
+
+    } else {
+      recordMutations.add(createRecordMutation(doc, extractRowId(doc)));
+    }
+    mutate.setRecordMutations(recordMutations);
+    return mutate;
+  }
+
+  private static String extractRowId(SolrInputDocument doc) {
+    Object id = doc.getFieldValue("id");
+    if (id == null) {
+      id = doc.getFieldValue("rowid");
+    }
+    if (id == null) {
+      throw new IllegalArgumentException("Document must either have id or rowid field.");
+    }
+    return id.toString();
+  }
+
+  private static RecordMutation createRecordMutation(SolrInputDocument doc, String id) {
+    RecordMutation recordMutation = new RecordMutation();
+    // TODO: what's solr default behavior?
+    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+    Record record = new Record();
+    record.setFamily(findFamily(doc));
+    record.setRecordId(id);
+
+    for (String fieldName : doc.getFieldNames()) {
+      if (!fieldName.contains(".")) {
+        continue;
+      }
+      SolrInputField field = doc.getField(fieldName);
+      String rawColumnName = fieldName.substring(fieldName.indexOf(".") + 1, fieldName.length());
+
+      if (field.getValueCount() > 1) {
+        for (Object fieldVal : field.getValues()) {
+          record.addToColumns(new Column(rawColumnName, fieldVal.toString()));
+        }
+      } else {
+        record.addToColumns(new Column(rawColumnName, field.getFirstValue().toString()));
+      }
+    }
+    recordMutation.setRecord(record);
+    return recordMutation;
+  }
+
+  private static String findFamily(SolrInputDocument doc) {
+    for (String name : doc.getFieldNames()) {
+      if (name.contains(".")) {
+        return name.substring(0, name.indexOf("."));
+      }
+    }
+    throw new IllegalArgumentException("Unable to determine column family from document");
+  }
+
+  private static void validate(SolrInputDocument doc) {
+    if ((doc.getFieldNames().size() > 1) && (doc.hasChildDocuments())) {
+
+    }
+
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/src/main/java/org/apache/blur/slur/SolrLookingBlurServer.java
----------------------------------------------------------------------
diff --git a/whiteboard/slur/src/main/java/org/apache/blur/slur/SolrLookingBlurServer.java
b/whiteboard/slur/src/main/java/org/apache/blur/slur/SolrLookingBlurServer.java
new file mode 100644
index 0000000..db2244d
--- /dev/null
+++ b/whiteboard/slur/src/main/java/org/apache/blur/slur/SolrLookingBlurServer.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.slur;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.BlurClientManager;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrRequest.METHOD;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.StreamingResponseCallback;
+import org.apache.solr.client.solrj.beans.DocumentObjectBinder;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.SolrPingResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+
+import com.google.common.collect.Lists;
+
+public class SolrLookingBlurServer extends SolrServer {
+  private static final long serialVersionUID = -7141309168500300896L;
+  private static final int DEFAULT_OPTIMIZE_MAX_SEGMENTS = 3;
+  private String connectionString;
+  private String tableName;
+
+  public SolrLookingBlurServer(String controllerConnectionStr, String table) {
+    connectionString = controllerConnectionStr;
+    tableName = table;
+  }
+
+  @Override
+  public NamedList<Object> request(SolrRequest arg0) throws SolrServerException, IOException
{
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public void shutdown() {
+    // Nothing to do
+  }
+
+  @Override
+  public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs)
throws SolrServerException,
+      IOException {
+    return add(docs);
+  }
+
+  @Override
+  public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException,
IOException {
+    UpdateResponse response = new UpdateResponse();
+
+    try {
+      long start = System.currentTimeMillis();
+      if (docs.size() == 1) {
+        client().mutate(RowMutationHelper.from(docs.iterator().next(), tableName));
+      } else {
+        client().mutateBatch(RowMutationHelper.from(docs, tableName));
+      }
+      response.setElapsedTime((System.currentTimeMillis() - start));
+    } catch (Exception e) {
+      throw new SolrServerException(e);
+    }
+
+    return response;
+  }
+
+  @Override
+  public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException,
IOException {
+    return add(doc);
+  }
+
+  @Override
+  public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException
{
+    return add(Lists.newArrayList(new SolrInputDocument[] { doc }));
+  }
+
+  @Override
+  public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException
{
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse addBean(Object obj) throws IOException, SolrServerException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse addBeans(Collection<?> arg0, int arg1) throws SolrServerException,
IOException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse addBeans(Collection<?> beans) throws SolrServerException, IOException
{
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse commit() throws SolrServerException, IOException {
+    return new UpdateResponse();
+  }
+
+  @Override
+  public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit)
throws SolrServerException,
+      IOException {
+    return new UpdateResponse();
+  }
+
+  @Override
+  public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException,
IOException {
+    return new UpdateResponse();
+  }
+
+  @Override
+  public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException,
IOException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException
{
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException,
IOException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse deleteById(String id) throws SolrServerException, IOException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException,
IOException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException
{
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public DocumentObjectBinder getBinder() {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse optimize() throws SolrServerException, IOException {
+    return optimize(true, true, DEFAULT_OPTIMIZE_MAX_SEGMENTS);
+  }
+
+  @Override
+  public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments)
throws SolrServerException,
+      IOException {
+    long start = System.currentTimeMillis();
+    try {
+      client().optimize(tableName, maxSegments);
+    } catch (Exception e) {
+      throw new SolrServerException(e);
+    }
+    UpdateResponse response = new UpdateResponse();
+    response.setElapsedTime((System.currentTimeMillis() - start));
+    return response;
+  }
+
+  @Override
+  public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException,
IOException {
+    return optimize(waitFlush, waitSearcher, DEFAULT_OPTIMIZE_MAX_SEGMENTS);
+  }
+
+  @Override
+  public SolrPingResponse ping() throws SolrServerException, IOException {
+    SolrPingResponse response = new SolrPingResponse();
+    long start = System.currentTimeMillis();
+    try {
+      client().ping();
+    } catch (TException e) {
+      throw new SolrServerException(e);
+    }
+    response.setElapsedTime((System.currentTimeMillis()-start));
+    return response;
+  }
+
+  @Override
+  public QueryResponse query(SolrParams params, METHOD method) throws SolrServerException
{
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public QueryResponse query(SolrParams params) throws SolrServerException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public QueryResponse queryAndStreamResponse(SolrParams params, StreamingResponseCallback
callback)
+      throws SolrServerException, IOException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  @Override
+  public UpdateResponse rollback() throws SolrServerException, IOException {
+    throw new RuntimeException("Not Implemented.");
+  }
+
+  private Iface client() {
+    return BlurClient.getClient(connectionString);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/src/test/java/org/apache/blur/slur/RowMutationHelperTest.java
----------------------------------------------------------------------
diff --git a/whiteboard/slur/src/test/java/org/apache/blur/slur/RowMutationHelperTest.java
b/whiteboard/slur/src/test/java/org/apache/blur/slur/RowMutationHelperTest.java
new file mode 100644
index 0000000..88a8b30
--- /dev/null
+++ b/whiteboard/slur/src/test/java/org/apache/blur/slur/RowMutationHelperTest.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.slur;
+
+import static org.junit.Assert.*;
+
+import java.util.List;
+
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class RowMutationHelperTest {
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+  }
+
+  @Test
+  public void basicOneForOneConversion() {
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField("fam.key", "value");
+    doc.addField("id", "123");
+
+    RowMutation mutate = RowMutationHelper.from(doc, "foo");
+
+    assertEquals("Should get our rowid back.", "123", mutate.getRowId());
+    assertEquals("Should get a single record.", 1, mutate.getRecordMutationsSize());
+    assertEquals("Should get a simple value back.", "value", getRecordValue("key", mutate));
+    assertEquals("Should properly figure our family.", "fam", getFirstRecord(mutate).getFamily());
+    assertEquals("Tablename should be set", "foo", mutate.getTable());
+  }
+
+  @Test
+  public void multivalueFieldsShouldTranslate() {
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField("id", "123");
+    doc.addField("fam.key", "value1");
+    doc.addField("fam.key", "value2");
+
+    RowMutation mutate = RowMutationHelper.from(doc, "foo");
+
+    List<Object> vals = getRecordValues("key", mutate);
+    assertEquals("Should get both values back.", 2, vals.size());
+
+    assertEquals("value1", vals.get(0).toString());
+    assertEquals("value2", vals.get(1).toString());
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void docWithChildrenCantItselfHaveFieldValues() {
+    SolrInputDocument parent = new SolrInputDocument();
+    parent.addField("fam.key1", "123");
+    SolrInputDocument child = new SolrInputDocument();
+    parent.addChildDocument(child);
+
+    RowMutationHelper.from(parent, "foo");
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void docsMustUseTheNormalBlurFamilyColumnFormat() {
+    SolrInputDocument parent = new SolrInputDocument();
+    parent.addField("columnWithoutFamily", "123");
+    SolrInputDocument child = new SolrInputDocument();
+    parent.addChildDocument(child);
+
+    RowMutationHelper.from(parent, "foo");
+  }
+
+  private Object getRecordValue(String field, RowMutation mutate) {
+    return getRecordValues(field, mutate).get(0);
+  }
+
+  private List<Object> getRecordValues(String field, RowMutation mutate) {
+    List<Object> vals = Lists.newArrayList();
+    Record rec = getFirstRecord(mutate);
+
+    for (Column col : rec.getColumns()) {
+      
+      if (col.getName().equals(field)) {
+        vals.add(col.getValue());
+      }
+    }
+    return vals;
+  }
+
+  private Record getFirstRecord(RowMutation mutate) {
+    return mutate.getRecordMutations().get(0).getRecord();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/src/test/java/org/apache/blur/slur/SolrLookingBlurServerTest.java
----------------------------------------------------------------------
diff --git a/whiteboard/slur/src/test/java/org/apache/blur/slur/SolrLookingBlurServerTest.java
b/whiteboard/slur/src/test/java/org/apache/blur/slur/SolrLookingBlurServerTest.java
new file mode 100644
index 0000000..401cdd7
--- /dev/null
+++ b/whiteboard/slur/src/test/java/org/apache/blur/slur/SolrLookingBlurServerTest.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.slur;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.blur.MiniCluster;
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.Query;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.thrift.generated.TableStats;
+import org.apache.blur.utils.GCWatcher;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class SolrLookingBlurServerTest {
+  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest"));
+  private static MiniCluster miniCluster;
+  private static String connectionStr;
+
+  @BeforeClass
+  public static void startCluster() throws IOException {
+    GCWatcher.init(0.60);
+    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
+    File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
+    testDirectory.mkdirs();
+
+    Path directory = new Path(testDirectory.getPath());
+    FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
+    FsAction userAction = dirPermissions.getUserAction();
+    FsAction groupAction = dirPermissions.getGroupAction();
+    FsAction otherAction = dirPermissions.getOtherAction();
+
+    StringBuilder builder = new StringBuilder();
+    builder.append(userAction.ordinal());
+    builder.append(groupAction.ordinal());
+    builder.append(otherAction.ordinal());
+    String dirPermissionNum = builder.toString();
+    System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
+    testDirectory.delete();
+    miniCluster = new MiniCluster();
+    miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2,
3, true);
+    connectionStr = miniCluster.getControllerConnectionStr();
+
+  }
+
+  @AfterClass
+  public static void shutdownCluster() {
+    miniCluster.shutdownBlurCluster();
+  }
+
+  @Test
+  public void addedDocumentShouldShowUpInBlur() throws SolrServerException, IOException,
BlurException, TException {
+    String table = "addedDocumentShouldShowUpInBlur";
+    createTable(table);
+    SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(),
table);
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField("id", "1");
+    doc.addField("fam.value", "123");
+
+    server.add(doc);
+
+    TableStats stats = client().tableStats(table);
+
+    assertEquals("We should have one record.", 1, stats.recordCount);
+    assertEquals("We should have one row.", 1, stats.rowCount);
+
+    assertTotalResults(table, "fam.value:123", 1l);
+
+    removeTable(table);
+  }
+
+  @Test
+  public void docShouldBeDiscoverableWithMultiValuedFields() throws SolrServerException,
IOException, BlurException,
+      TException {
+    String table = "docShouldBeDiscoverableWithMultiValuedFields";
+    createTable(table);
+    SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(),
table);
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField("id", "1");
+    doc.addField("fam.value", "123");
+    doc.addField("fam.value", "124");
+
+    server.add(doc);
+
+    assertTotalResults(table, "fam.value:123", 1l);
+    assertTotalResults(table, "fam.value:124", 1l);
+    assertTotalResults(table, "fam.value:justincase", 0l);
+
+    removeTable(table);
+  }
+
+  @Test
+  public void documentsShouldBeAbleToBeIndexedInBatch() throws SolrServerException, IOException,
BlurException,
+      TException {
+    String table = "multipleDocumentsShouldBeIndexed";
+    createTable(table);
+
+    SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(),
table);
+
+    List<SolrInputDocument> docs = Lists.newArrayList();
+
+    for (int i = 0; i < 100; i++) {
+      SolrInputDocument doc = new SolrInputDocument();
+      doc.addField("id", i);
+      doc.addField("fam.value", "12" + i);
+      docs.add(doc);
+    }
+
+    server.add(docs);
+
+    assertTotalResults(table, "fam.value:123", 1l);
+    assertTotalResults(table, "fam.value:124", 1l);
+    assertTotalResults(table, "rowid:1", 1l);
+    assertTotalResults(table, "rowid:2", 1l);
+    assertTotalResults(table, "fam.value:1299", 1l);
+    assertTotalResults(table, "fam.value:justincase", 0l);
+    
+    removeTable(table);
+  }
+
+  private void assertTotalResults(String table, String q, long expected) throws BlurException,
TException {
+    BlurQuery bquery = new BlurQuery();
+    Query query = new Query();
+    query.setQuery(q);
+    bquery.setQuery(query);
+    BlurResults results = client().query(table, bquery);
+
+    assertEquals("Should find our row.", expected, results.getTotalResults());
+  }
+
+  private void removeTable(String table) {
+    try {
+      client().disableTable(table);
+    } catch (Exception e) {
+
+    }
+    try {
+      client().removeTable(table, true);
+    } catch (Exception e) {
+
+    }
+  }
+
+  private static void createTable(String tableName) throws BlurException, TException, IOException
{
+    Blur.Iface client = client();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName(tableName);
+    tableDescriptor.setShardCount(5);
+    tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName);
+    client.createTable(tableDescriptor);
+    List<String> tableList = client.tableList();
+    assertTrue(tableList.contains(tableName));
+  }
+
+  private static Iface client() {
+    return BlurClient.getClient(connectionStr);
+  }
+
+}


Mime
View raw message