incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [7/8] git commit: Now the update documents call works.
Date Sun, 25 Nov 2012 02:34:11 GMT
Now the update documents call works.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/64943aa0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/64943aa0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/64943aa0

Branch: refs/heads/0.2-dev-removing-old-thrift
Commit: 64943aa0bef4cfc93595b47e0277ccbafe571bf9
Parents: 8018b06
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Nov 24 20:46:19 2012 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Nov 24 20:46:19 2012 -0500

----------------------------------------------------------------------
 .../org/apache/blur/thrift/BlurShardServer.java    |   19 +-
 .../org/apache/blur/thrift/BlurClusterTest.java    |  137 ------
 .../blur/thrift/BlurControllerServerTest.java      |  334 ---------------
 .../apache/blur/thrift/BlurShardServerTest.java    |  151 -------
 .../org/apache/blur/thrift/DoNothingServer.java    |  134 ------
 .../org/apache/blur/thrift/ITBlurClusterTest.java  |  186 ++++++++
 6 files changed, 204 insertions(+), 757 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
index 5526f85..088370f 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
@@ -426,6 +426,19 @@ public class BlurShardServer extends TableAdmin implements Iface {
       }
     });
   }
+  
+  private void forwardUpdateDocuments(final MutateOptions options, final List<UpdatePackage>
updatePackages) throws BlurException, TException, IOException {
+    String table = options.getTable();
+    int shardIndex = options.getShardIndex();
+    Connection connection = getConnection(table, shardIndex);
+    BlurClientManager.execute(connection, new BlurCommand<Void>() {
+      @Override
+      public Void call(Client client) throws BlurException, TException {
+        client.updateDocuments(options, updatePackages);
+        return null;
+      }
+    });
+  }
 
   private Connection getConnection(String table, int shardIndex) {
     String server = _layout.findServer(table, shardIndex, TYPE.WRITABLE);
@@ -470,7 +483,11 @@ public class BlurShardServer extends TableAdmin implements Iface {
     boolean writeAheadLog = options.isWriteAheadLog();
     try {
       BlurIndex index = getIndex(table, shardIndex);
-      index.updateDocuments(waitToBeVisible, writeAheadLog, updatePackages);
+      if (index == null) {
+        forwardUpdateDocuments(options, updatePackages);
+      } else {
+        index.updateDocuments(waitToBeVisible, writeAheadLog, updatePackages);
+      }
     } catch (Throwable t) {
       LOG.error("Unknown error", t);
       throw new BException(t.getMessage(), t);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
deleted file mode 100644
index de97d6d..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ /dev/null
@@ -1,137 +0,0 @@
-package org.apache.blur.thrift;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.blur.MiniCluster;
-import org.apache.blur.thrift.generated.Blur;
-import org.apache.blur.thrift.generated.Blur.Iface;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.Document;
-import org.apache.blur.thrift.generated.Field;
-import org.apache.blur.thrift.generated.MutateOptions;
-import org.apache.blur.thrift.generated.QueryArgs;
-import org.apache.blur.thrift.generated.Session;
-import org.apache.blur.thrift.generated.TYPE;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.thrift.generated.TopFieldDocs;
-import org.apache.blur.thrift.lucene.Convert;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.TermQuery;
-import org.apache.thrift.TException;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class BlurClusterTest {
-
-  private static final int SHARD_COUNT = 5;
-  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp"));
-  private static final int SHARD_SERVERS = 2;
-
-  @BeforeClass
-  public static void startCluster() throws IOException {
-    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
-    File testDirectory = new File(TMPDIR, "blur-cluster-test");
-    testDirectory.mkdirs();
-
-    Path directory = new Path(testDirectory.getPath());
-    FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
-    FsAction userAction = dirPermissions.getUserAction();
-    FsAction groupAction = dirPermissions.getGroupAction();
-    FsAction otherAction = dirPermissions.getOtherAction();
-
-    StringBuilder builder = new StringBuilder();
-    builder.append(userAction.ordinal());
-    builder.append(groupAction.ordinal());
-    builder.append(otherAction.ordinal());
-    String dirPermissionNum = builder.toString();
-    System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
-    testDirectory.delete();
-
-    MiniCluster.startBlurCluster("cluster", SHARD_SERVERS);
-  }
-
-  @AfterClass
-  public static void shutdownCluster() {
-    MiniCluster.shutdownBlurCluster();
-  }
-
-  @Test
-  public void testCreateTable() throws BlurException, TException, IOException {
-    Blur.Iface client = getClient();
-    TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.setName("test");
-    tableDescriptor.setShardCount(SHARD_COUNT);
-    tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test");
-    client.createTable(tableDescriptor);
-    List<String> tableList = client.tableList();
-    assertEquals(Arrays.asList("test"), tableList);
-  }
-
-  private Iface getClient() {
-    return BlurClient.getClient(MiniCluster.getConnectionStr(), 0, 0, 0);
-  }
-
-  @Test
-  public void testLoadTable() throws BlurException, TException, InterruptedException, IOException
{
-    Iface client = getClient();
-    for (int s = 0; s < SHARD_COUNT; s++) {
-      int length = 100;
-      List<Document> documents = new ArrayList<Document>();
-      for (int i = 0; i < length; i++) {
-        Document doc = new Document();
-        doc.addToFields(new Field("id", ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()),
TYPE.STRING, 1.0));
-        doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING,
1.0));
-        documents.add(doc);
-      }
-      MutateOptions options = new MutateOptions();
-      options.setTable("test");
-      options.setWaitToBeVisible(true);
-      options.setShardIndex(s);
-      client.addDocuments(options, documents);
-    }
-
-    Session session = client.openReadSession("test");
-    QueryArgs queryArgs = new QueryArgs();
-    Term term = new Term("value", "test");
-    TermQuery query = new TermQuery(term);
-    queryArgs.setQuery(Convert.toBytes(query));
-    List<TopFieldDocs> results = client.search(session, queryArgs);
-    long totalHits = 0;
-    for (TopFieldDocs fieldDocs : results) {
-      totalHits += fieldDocs.getTotalHits();
-    }
-    assertEquals(SHARD_COUNT * 100, totalHits);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/BlurControllerServerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurControllerServerTest.java
b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurControllerServerTest.java
deleted file mode 100644
index dd2f777..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurControllerServerTest.java
+++ /dev/null
@@ -1,334 +0,0 @@
-package org.apache.blur.thrift;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.junit.Test;
-
-public class BlurControllerServerTest {
-
-  @Test
-  public void testNothing() {
-    // do nothing
-  }
-
-  // private static final String TABLE = "test";
-  // private static Map<String, Iface> shardServers = new HashMap<String,
-  // Iface>();
-  // private static BlurControllerServer server;
-  // private static Thread daemonService;
-  //
-  // @BeforeClass
-  // public static void zkServerStartup() throws InterruptedException,
-  // KeeperException, IOException, BlurException, TException {
-  // rm(new File(TMPDIR, "../zk-tmp"));
-  // daemonService = new Thread(new Runnable() {
-  // @Override
-  // public void run() {
-  // QuorumPeerMain.main(new
-  // String[]{"./src/test/resources/com/nearinfinity/blur/thrift/zoo.cfg"});
-  // }
-  // });
-  // daemonService.start();
-  // Thread.sleep(1000);
-  //
-  // ZooKeeper zookeeper = new ZooKeeper("127.0.0.1:10101", 30000, new Watcher()
-  // {
-  // @Override
-  // public void process(WatchedEvent event) {
-  //
-  // }
-  // });
-  //
-  // addShardServer("shard-00000000");
-  // addShardServer("shard-00000001");
-  // addShardServer("shard-00000002");
-  // BlurQueryChecker queryChecker = new BlurQueryChecker(new
-  // BlurConfiguration());
-  // server = new BlurControllerServer();
-  // server.setClient(getClient());
-  // server.setClusterStatus(new ZookeeperClusterStatus(zookeeper));
-  // server.setQueryChecker(queryChecker);
-  // server.setZookeeper(zookeeper);
-  // server.init();
-  //
-  // File file = new File(TMPDIR, "./tmp-data/test");
-  // rm(file);
-  //
-  // TableDescriptor tableDescriptor = new TableDescriptor();
-  // tableDescriptor.name = "test";
-  // tableDescriptor.shardCount = 3;
-  // tableDescriptor.tableUri = file.toURI().toString();
-  // tableDescriptor.analyzerDefinition = new AnalyzerDefinition();
-  // server.createTable(tableDescriptor);
-  // }
-  //
-  // @AfterClass
-  // public static void zkServerShutdown() {
-  // daemonService.interrupt();
-  // server.close();
-  // }
-  //
-  // private static void rm(File file) {
-  // if (file.isDirectory()) {
-  // for (File f : file.listFiles()) {
-  // rm(f);
-  // }
-  // }
-  // file.delete();
-  // }
-  //
-  // @Test
-  // public void testQuery() throws BlurException, TException {
-  // BlurQuery blurQuery = new BlurQuery();
-  // blurQuery.maxQueryTime = TimeUnit.SECONDS.toMillis(5);
-  // blurQuery.minimumNumberOfResults = Long.MAX_VALUE;
-  // BlurResults results = server.query(TABLE, blurQuery);
-  // assertNotNull(results);
-  // }
-  //
-  // @Test
-  // public void testQueryWithFacets() throws BlurException, TException {
-  // BlurQuery blurQuery = new BlurQuery();
-  // blurQuery.maxQueryTime = TimeUnit.SECONDS.toMillis(5);
-  // blurQuery.minimumNumberOfResults = Long.MAX_VALUE;
-  // blurQuery.facets = new ArrayList<Facet>();
-  // blurQuery.facets.add(new Facet());
-  // blurQuery.facets.add(new Facet());
-  // BlurResults results = server.query(TABLE, blurQuery);
-  // assertNotNull(results);
-  // assertNotNull(results.facetCounts);
-  // for (int i = 0; i < results.facetCounts.size(); i++) {
-  // long count = results.facetCounts.get(i);
-  // assertEquals(shardServers.size() * (i + 1), count);
-  // }
-  // }
-  //
-  // @Test
-  // public void testRecordFrequency() throws BlurException, TException {
-  // long recordFrequency = server.recordFrequency(TABLE, "cf", "cn", "value");
-  // assertEquals(shardServers.size(), recordFrequency);
-  // }
-  //
-  // @Test
-  // public void testMutate() throws BlurException, TException {
-  // RowMutation mutation = new RowMutation();
-  // mutation.setRowId("1234");
-  // RecordMutation recMut = new RecordMutation();
-  // Record record = new Record();
-  // record.setFamily("test");
-  // record.setRecordId("5678");
-  // record.addToColumns(new Column("name", "value"));
-  // mutation.addToRecordMutations(recMut);
-  // mutation.table = TABLE;
-  // server.mutate(mutation);
-  //
-  // Selector selector = new Selector();
-  // selector.rowId = "1234";
-  //
-  // FetchResult fetchRow = server.fetchRow(TABLE, selector);
-  // assertNotNull(fetchRow.rowResult);
-  // }
-  //
-  // private static BlurClient getClient() {
-  // BlurClientEmbedded blurClientEmbedded = new BlurClientEmbedded();
-  // for (String node : shardServers.keySet()) {
-  // blurClientEmbedded.putNode(node, shardServers.get(node));
-  // }
-  // return blurClientEmbedded;
-  // }
-  //
-  // private static Iface getShardServer(final String node) {
-  // return new Iface() {
-  //
-  // private Map<String, Map<String, Row>> rows = new HashMap<String,
-  // Map<String, Row>>();
-  //
-  // @Override
-  // public List<String> terms(String arg0, String arg1, String arg2, String
-  // arg3, short arg4) throws BlurException, TException {
-  // throw new RuntimeException("no impl");
-  // }
-  //
-  // @Override
-  // public List<String> tableList() throws BlurException, TException {
-  // List<String> table = new ArrayList<String>();
-  // table.add(TABLE);
-  // return table;
-  // }
-  //
-  // @Override
-  // public List<String> shardServerList(String cluster) throws BlurException,
-  // TException {
-  // throw new RuntimeException("no impl");
-  // }
-  //
-  // @Override
-  // public Map<String, String> shardServerLayout(String table) throws
-  // BlurException, TException {
-  // Map<String, String> layout = new HashMap<String, String>();
-  // layout.put(node, node);
-  // return layout;
-  // }
-  //
-  // @Override
-  // public BlurResults query(String table, BlurQuery query) throws
-  // BlurException, TException {
-  // BlurResults results = new BlurResults();
-  // results.putToShardInfo(node, 0);
-  // results.setFacetCounts(getFacetCounts(query));
-  // return results;
-  // }
-  //
-  // private List<Long> getFacetCounts(BlurQuery query) {
-  // if (query.facets != null) {
-  // int size = query.facets.size();
-  // List<Long> results = new ArrayList<Long>();
-  // for (int i = 0; i < size; i++) {
-  // results.add(i + 1L);
-  // }
-  // return results;
-  // }
-  // return null;
-  // }
-  //
-  // @Override
-  // public Schema schema(String arg0) throws BlurException, TException {
-  // throw new RuntimeException("no impl");
-  // }
-  //
-  // @Override
-  // public long recordFrequency(String arg0, String arg1, String arg2, String
-  // arg3) throws BlurException, TException {
-  // return 1l;
-  // }
-  //
-  // @Override
-  // public FetchResult fetchRow(String table, Selector selector) throws
-  // BlurException, TException {
-  // Map<String, Row> map = rows.get(table);
-  // Row row = map.get(selector.rowId);
-  // FetchResult fetchResult = new FetchResult();
-  // fetchResult.setRowResult(new FetchRowResult(row));
-  // return fetchResult;
-  // }
-  //
-  // @Override
-  // public TableDescriptor describe(String arg0) throws BlurException,
-  // TException {
-  // TableDescriptor descriptor = new TableDescriptor();
-  // descriptor.isEnabled = true;
-  // descriptor.shardCount = 3;
-  // return descriptor;
-  // }
-  //
-  // @Override
-  // public List<BlurQueryStatus> currentQueries(String arg0) throws
-  // BlurException, TException {
-  // throw new RuntimeException("no impl");
-  // }
-  //
-  // @Override
-  // public List<String> controllerServerList() throws BlurException, TException
-  // {
-  // throw new RuntimeException("no impl");
-  // }
-  //
-  // @Override
-  // public void cancelQuery(String table, long arg0) throws BlurException,
-  // TException {
-  // throw new RuntimeException("no impl");
-  // }
-  //
-  // private Row toRow(RowMutation mutation) {
-  // Row row = new Row();
-  // row.id = mutation.rowId;
-  // row.records = toRecords(mutation.recordMutations);
-  // return row;
-  // }
-  //
-  // private List<Record> toRecords(List<RecordMutation> recordMutations) {
-  // List<Record> records = new ArrayList<Record>();
-  // for (RecordMutation mutation : recordMutations) {
-  // records.add(mutation.record);
-  // }
-  // return records;
-  // }
-  //
-  // @Override
-  // public void createTable(TableDescriptor tableDescriptor) throws
-  // BlurException, TException {
-  // throw new RuntimeException("not impl");
-  // }
-  //
-  // @Override
-  // public void disableTable(String table) throws BlurException, TException {
-  // throw new RuntimeException("not impl");
-  // }
-  //
-  // @Override
-  // public void enableTable(String table) throws BlurException, TException {
-  // throw new RuntimeException("not impl");
-  // }
-  //
-  // @Override
-  // public void removeTable(String table, boolean deleteIndexFiles) throws
-  // BlurException, TException {
-  // throw new RuntimeException("not impl");
-  // }
-  //
-  // @Override
-  // public TableStats getTableStats(String table) throws BlurException,
-  // TException {
-  // return new TableStats();
-  // }
-  //
-  // @Override
-  // public void mutate(RowMutation mutation) throws BlurException, TException {
-  // String table = mutation.table;
-  // Map<String, Row> map = rows.get(table);
-  // if (map == null) {
-  // map = new HashMap<String, Row>();
-  // rows.put(table, map);
-  // }
-  // Row row = toRow(mutation);
-  // map.put(row.id, row);
-  // }
-  //
-  // @Override
-  // public void mutateBatch(List<RowMutation> mutations) throws BlurException,
-  // TException {
-  // for (RowMutation mutation : mutations) {
-  // MutationHelper.validateMutation(mutation);
-  // }
-  // for (RowMutation mutation : mutations) {
-  // mutate(mutation);
-  // }
-  // }
-  //
-  // @Override
-  // public List<String> shardClusterList() throws BlurException, TException {
-  // throw new RuntimeException("no impl");
-  // }
-  // };
-  // }
-  //
-  // private static void addShardServer(String node) {
-  // shardServers.put(node, getShardServer(node));
-  // }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/BlurShardServerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurShardServerTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurShardServerTest.java
deleted file mode 100644
index 7f78ba4..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurShardServerTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-package org.apache.blur.thrift;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedSet;
-
-import org.apache.blur.BlurConfiguration;
-import org.apache.blur.analysis.BlurAnalyzer;
-import org.apache.blur.manager.IndexManager;
-import org.apache.blur.manager.IndexServer;
-import org.apache.blur.manager.writer.BlurIndex;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.lucene.search.similarities.Similarity;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class BlurShardServerTest {
-
-  private BlurShardServer blurShardServer;
-
-  @Before
-  public void setUp() throws BlurException, IOException {
-    IndexServer indexServer = getIndexServer();
-    IndexManager indexManager = getIndexManager();
-    indexManager.setIndexServer(indexServer);
-    indexManager.setThreadCount(1);
-    indexManager.init();
-    blurShardServer = new BlurShardServer();
-    blurShardServer.setIndexManager(indexManager);
-    blurShardServer.setIndexServer(indexServer);
-    blurShardServer.setConfiguration(new BlurConfiguration());
-    blurShardServer.init();
-  }
-
-  @After
-  public void tearDown() throws InterruptedException {
-    blurShardServer.close();
-  }
-
-  @Test
-  public void testNothingYet() {
-
-  }
-
-  private IndexServer getIndexServer() {
-    return new IndexServer() {
-
-      @Override
-      public void close() {
-
-      }
-
-      @Override
-      public BlurAnalyzer getAnalyzer(String table) {
-        throw new RuntimeException("not impl");
-      }
-
-      @Override
-      public Map<String, BlurIndex> getIndexes(String table) throws IOException {
-        throw new RuntimeException("not impl");
-      }
-
-      @Override
-      public List<String> getShardList(String table) {
-        throw new RuntimeException("not impl");
-      }
-
-      @Override
-      public Similarity getSimilarity(String table) {
-        throw new RuntimeException("not impl");
-      }
-
-      @Override
-      public TABLE_STATUS getTableStatus(String table) {
-        throw new RuntimeException("not impl");
-      }
-
-      @Override
-      public String getNodeName() {
-        throw new RuntimeException("no impl");
-      }
-
-      @Override
-      public String getTableUri(String table) {
-        throw new RuntimeException("no impl");
-      }
-
-      @Override
-      public int getShardCount(String table) {
-        throw new RuntimeException("no impl");
-      }
-
-      @Override
-      public int getCompressionBlockSize(String table) {
-        throw new RuntimeException("no impl");
-      }
-
-      @Override
-      public CompressionCodec getCompressionCodec(String table) {
-        throw new RuntimeException("no impl");
-      }
-
-      @Override
-      public long getRecordCount(String table) throws IOException {
-        throw new RuntimeException("no impl");
-      }
-
-      @Override
-      public long getRowCount(String table) throws IOException {
-        throw new RuntimeException("no impl");
-      }
-
-      @Override
-      public long getTableSize(String table) throws IOException {
-        throw new RuntimeException("no impl");
-      }
-
-      @Override
-      public SortedSet<String> getShardListCurrentServerOnly(String table) throws IOException
{
-        throw new RuntimeException("no impl");
-      }
-    };
-  }
-
-  private IndexManager getIndexManager() {
-    return new IndexManager() {
-
-    };
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/DoNothingServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/DoNothingServer.java b/src/blur-core/src/test/java/org/apache/blur/thrift/DoNothingServer.java
deleted file mode 100644
index e9f1a7a..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/thrift/DoNothingServer.java
+++ /dev/null
@@ -1,134 +0,0 @@
-package org.apache.blur.thrift;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.blur.thrift.generated.Blur.Iface;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.BlurQueryStatus;
-import org.apache.blur.thrift.generated.Schema;
-import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.thrift.generated.TableStats;
-import org.apache.thrift.TException;
-
-
-public class DoNothingServer implements Iface {
-
-  @Override
-  public TableDescriptor describe(String table) throws BlurException, TException {
-    return null;
-  }
-
-  @Override
-  public List<String> tableList() throws BlurException, TException {
-    return Arrays.asList("donothing");
-  }
-
-  @Override
-  public List<String> controllerServerList() throws BlurException, TException {
-    return null;
-  }
-
-  @Override
-  public List<String> shardServerList(String cluster) throws BlurException, TException
{
-    return null;
-  }
-
-  @Override
-  public Map<String, String> shardServerLayout(String table) throws BlurException,
TException {
-    return null;
-  }
-
-  @Override
-  public void cancelQuery(String table, long providedUuid) throws BlurException, TException
{
-
-  }
-
-  @Override
-  public List<BlurQueryStatus> currentQueries(String arg0) throws BlurException, TException
{
-    return null;
-  }
-
-  @Override
-  public Schema schema(String arg0) throws BlurException, TException {
-    return null;
-  }
-
-  @Override
-  public void createTable(TableDescriptor tableDescriptor) throws BlurException, TException
{
-
-  }
-
-  @Override
-  public void disableTable(String table) throws BlurException, TException {
-
-  }
-
-  @Override
-  public void enableTable(String table) throws BlurException, TException {
-
-  }
-
-  @Override
-  public void removeTable(String table, boolean deleteIndexFiles) throws BlurException, TException
{
-
-  }
-
-  @Override
-  public List<String> shardClusterList() throws BlurException, TException {
-    return null;
-  }
-
-  @Override
-  public List<String> tableListByCluster(String cluster) throws BlurException, TException
{
-    return null;
-  }
-
-  @Override
-  public BlurQueryStatus queryStatusById(String table, long uuid) throws BlurException, TException
{
-    return null;
-  }
-
-  @Override
-  public List<Long> queryStatusIdList(String table) throws BlurException, TException
{
-    return null;
-  }
-
-  @Override
-  public void optimize(String table, int numberOfSegmentsPerShard) throws BlurException,
TException {
-
-  }
-
-  @Override
-  public boolean isInSafeMode(String cluster) throws BlurException, TException {
-    return false;
-  }
-
-  @Override
-  public TableStats tableStats(String table) throws BlurException, TException {
-    return null;
-  }
-
-  @Override
-  public Map<String, String> configuration() throws BlurException, TException {
-    return null;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
new file mode 100644
index 0000000..dd0e4b1
--- /dev/null
+++ b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
@@ -0,0 +1,186 @@
+package org.apache.blur.thrift;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.blur.MiniCluster;
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Document;
+import org.apache.blur.thrift.generated.Field;
+import org.apache.blur.thrift.generated.MutateOptions;
+import org.apache.blur.thrift.generated.QueryArgs;
+import org.apache.blur.thrift.generated.Session;
+import org.apache.blur.thrift.generated.TYPE;
+import org.apache.blur.thrift.generated.TableDescriptor;
+import org.apache.blur.thrift.generated.TopFieldDocs;
+import org.apache.blur.thrift.generated.UpdatePackage;
+import org.apache.blur.thrift.lucene.Convert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.TermQuery;
+import org.apache.thrift.TException;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ITBlurClusterTest {
+
+  private static final int SHARD_COUNT = 5;
+  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp"));
+  private static final int SHARD_SERVERS = 2;
+
+  @BeforeClass
+  public static void startCluster() throws IOException {
+    LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
+    File testDirectory = new File(TMPDIR, "blur-cluster-test");
+    testDirectory.mkdirs();
+
+    Path directory = new Path(testDirectory.getPath());
+    FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
+    FsAction userAction = dirPermissions.getUserAction();
+    FsAction groupAction = dirPermissions.getGroupAction();
+    FsAction otherAction = dirPermissions.getOtherAction();
+
+    StringBuilder builder = new StringBuilder();
+    builder.append(userAction.ordinal());
+    builder.append(groupAction.ordinal());
+    builder.append(otherAction.ordinal());
+    String dirPermissionNum = builder.toString();
+    System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
+    testDirectory.delete();
+
+    MiniCluster.startBlurCluster("cluster", SHARD_SERVERS);
+  }
+
+  @AfterClass
+  public static void shutdownCluster() {
+    MiniCluster.shutdownBlurCluster();
+  }
+
+  @Test
+  public void testCreateTable() throws BlurException, TException, IOException {
+    Blur.Iface client = getClient();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName("test");
+    tableDescriptor.setShardCount(SHARD_COUNT);
+    tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test");
+    client.createTable(tableDescriptor);
+    List<String> tableList = client.tableList();
+    assertEquals(Arrays.asList("test"), tableList);
+  }
+
+  private Iface getClient() {
+    return BlurClient.getClient(MiniCluster.getConnectionStr(), 0, 0, 0);
+  }
+
+  @Test
+  public void testAddDocuments() throws BlurException, TException, InterruptedException,
IOException {
+    Iface client = getClient();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName("test-add-table");
+    tableDescriptor.setShardCount(SHARD_COUNT);
+    tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test-add-table");
+    client.createTable(tableDescriptor);
+    
+    for (int s = 0; s < SHARD_COUNT; s++) {
+      int length = 100;
+      List<Document> documents = new ArrayList<Document>();
+      for (int i = 0; i < length; i++) {
+        Document doc = new Document();
+        doc.addToFields(new Field("id", ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()),
TYPE.STRING, 1.0));
+        doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING,
1.0));
+        documents.add(doc);
+      }
+      MutateOptions options = new MutateOptions();
+      options.setTable(tableDescriptor.getName());
+      options.setWaitToBeVisible(true);
+      options.setShardIndex(s);
+      client.addDocuments(options, documents);
+    }
+
+    Session session = client.openReadSession(tableDescriptor.getName());
+    QueryArgs queryArgs = new QueryArgs();
+    Term term = new Term("value", "test");
+    TermQuery query = new TermQuery(term);
+    queryArgs.setQuery(Convert.toBytes(query));
+    List<TopFieldDocs> results = client.search(session, queryArgs);
+    long totalHits = 0;
+    for (TopFieldDocs fieldDocs : results) {
+      totalHits += fieldDocs.getTotalHits();
+    }
+    assertEquals(SHARD_COUNT * 100, totalHits);
+  }
+  
+  @Test
+  public void testUpdateDocuments() throws BlurException, TException, InterruptedException,
IOException {
+    Iface client = getClient();
+    TableDescriptor tableDescriptor = new TableDescriptor();
+    tableDescriptor.setName("test-update-table");
+    tableDescriptor.setShardCount(SHARD_COUNT);
+    tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test-update-table");
+    client.createTable(tableDescriptor);
+    
+    for (int s = 0; s < SHARD_COUNT; s++) {
+      int length = 100;
+      List<UpdatePackage> updatePackages = new ArrayList<UpdatePackage>();
+      for (int i = 0; i < length; i++) {
+        UpdatePackage updatePackage = new UpdatePackage();
+        Document doc = new Document();
+        byte[] uuid = UUID.randomUUID().toString().getBytes();
+        doc.addToFields(new Field("id", ByteBuffer.wrap(uuid), TYPE.STRING, 1.0));
+        doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING,
1.0));
+        updatePackage.setTerm(new org.apache.blur.thrift.generated.Term("id", ByteBuffer.wrap(uuid)));
+        updatePackage.addToDocuments(doc);
+        updatePackages.add(updatePackage);
+      }
+      MutateOptions options = new MutateOptions();
+      options.setTable(tableDescriptor.getName());
+      options.setWaitToBeVisible(true);
+      options.setShardIndex(s);
+      client.updateDocuments(options, updatePackages);
+    }
+
+    Session session = client.openReadSession(tableDescriptor.getName());
+    QueryArgs queryArgs = new QueryArgs();
+    Term term = new Term("value", "test");
+    TermQuery query = new TermQuery(term);
+    queryArgs.setQuery(Convert.toBytes(query));
+    List<TopFieldDocs> results = client.search(session, queryArgs);
+    long totalHits = 0;
+    for (TopFieldDocs fieldDocs : results) {
+      totalHits += fieldDocs.getTotalHits();
+    }
+    assertEquals(SHARD_COUNT * 100, totalHits);
+  }
+}


Mime
View raw message