incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [12/12] git commit: Modified blur service and added the snapshot methods, generated the code, hooked the controller and shard servers for the new methods and stubbed the BlurNRTIndex
Date Sat, 17 Aug 2013 16:55:52 GMT
Modified blur service and added the snapshot methods, generated the code, hooked the controller
and shard servers for the new methods and stubbed the BlurNRTIndex

Signed-off-by: Aaron McCurry <amccurry@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d512053c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d512053c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d512053c

Branch: refs/heads/master
Commit: d512053c53e65182c184204a6b1fd720e4c37320
Parents: 079113a
Author: Rahul Challapalli <challapallirahul@gmail.com>
Authored: Fri Aug 16 23:19:08 2013 -0700
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Aug 17 12:53:33 2013 -0400

----------------------------------------------------------------------
 .../apache/blur/manager/writer/BlurIndex.java   |    7 +
 .../blur/manager/writer/BlurIndexReader.java    |   17 +
 .../blur/manager/writer/BlurNRTIndex.java       |   16 +
 .../blur/thrift/BlurControllerServer.java       |   68 +
 .../org/apache/blur/thrift/BlurShardServer.java |   63 +
 .../org/apache/blur/thrift/generated/Blur.java  | 2809 ++++++++++++++++++
 .../src/main/scripts/interface/Blur.thrift      |   14 +
 .../main/scripts/interface/gen-html/Blur.html   |   19 +-
 .../main/scripts/interface/gen-html/index.html  |    3 +
 .../org/apache/blur/thrift/generated/Blur.java  | 2809 ++++++++++++++++++
 .../src/main/scripts/interface/gen-js/Blur.js   |  533 ++++
 .../scripts/interface/gen-perl/Blur/Blur.pm     |  713 +++++
 .../src/main/scripts/interface/gen-rb/blur.rb   |  181 ++
 13 files changed, 7251 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
index 344e886..19b26b6 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java
@@ -17,6 +17,7 @@ package org.apache.blur.manager.writer;
  * limitations under the License.
  */
 import java.io.IOException;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -51,6 +52,12 @@ public abstract class BlurIndex {
   public abstract AtomicBoolean isClosed();
 
   public abstract void optimize(int numberOfSegmentsPerShard) throws IOException;
+  
+  public abstract void createSnapshot(String name) throws IOException;
+  
+  public abstract void removeSnapshot(String name) throws IOException;
+  
+  public abstract List<String> getSnapshots() throws IOException;
 
   public long getRecordCount() throws IOException {
     IndexSearcherClosable searcher = getIndexReader();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
index f3733e6..6017e0a 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
@@ -19,6 +19,7 @@ package org.apache.blur.manager.writer;
 import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -29,6 +30,7 @@ import org.apache.blur.lucene.warmup.TraceableDirectory;
 import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
+import org.apache.blur.thrift.BException;
 import org.apache.blur.thrift.generated.Row;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.index.BlurIndexWriter;
@@ -130,4 +132,19 @@ public class BlurIndexReader extends BlurIndex {
   public AtomicBoolean isClosed() {
     return _isClosed;
   }
+
+  @Override
+  public void createSnapshot(String name) throws IOException {
+    throw new RuntimeException("Read-only shard");
+  }
+
+  @Override
+  public void removeSnapshot(String name) throws IOException {
+    throw new RuntimeException("Read-only shard");
+  }
+
+  @Override
+  public List<String> getSnapshots() throws IOException {
+    throw new RuntimeException("Read-only shard");
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index 993074a..ed0569a 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -264,4 +264,20 @@ public class BlurNRTIndex extends BlurIndex {
     }
   }
 
+  @Override
+  public void createSnapshot(String name) throws IOException {
+    // TODO Auto-generated method stub
+  }
+
+  @Override
+  public void removeSnapshot(String name) throws IOException {
+    // TODO Auto-generated method stub
+  }
+
+  @Override
+  public List<String> getSnapshots() throws IOException {
+    // TODO Auto-generated method stub
+    return null;
+  }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index cdac2f1..822508c 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -830,6 +830,74 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
       }
     }
   }
+  
+  @Override
+  public void createSnapshot(final String table, final String name) throws BlurException,
TException {
+    checkTable(table);
+    try {
+      scatter(getCluster(table), new BlurCommand<Void>() {
+        @Override
+        public Void call(Client client) throws BlurException, TException {
+          client.createSnapshot(table, name);
+          return null;
+        }
+      });
+    } catch (Exception e) {
+      LOG.error("Unknown error while trying to create a snapshot of table [{0}] snapshot
name", e, table, name);
+      throw new BException("Unknown error while trying to create a snapshot of table [{0}]
snapshot name", e, table, name);
+    }
+  }
+  
+  @Override
+  public void removeSnapshot(final String table, final String name) throws BlurException,
TException {
+    checkTable(table);
+    try {
+      scatter(getCluster(table), new BlurCommand<Void>() {
+        @Override
+        public Void call(Client client) throws BlurException, TException {
+          client.removeSnapshot(table, name);
+          return null;
+        }
+      });
+    } catch (Exception e) {
+      LOG.error("Unknown error while trying to remove a snapshot of table [{0}] snapshot
name", e, table, name);
+      throw new BException("Unknown error while trying to remove a snapshot of table [{0}]
snapshot name", e, table, name);
+    }
+  }
+  
+  @Override
+  public Map<String, List<String>> listSnapshots(final String table) throws BlurException,
TException {
+    checkTable(table);
+    try {
+      return scatterGather(getCluster(table), new BlurCommand<Map<String, List<String>>>()
{
+        @Override
+        public Map<String, List<String>> call(Client client) throws BlurException,
TException {
+          return client.listSnapshots(table);
+        }
+      }, new Merger<Map<String, List<String>>>() {
+        @Override
+        public Map<String, List<String>> merge(BlurExecutorCompletionService<Map<String,
List<String>>> service) throws BlurException {
+          Map<String, List<String>> result = new HashMap<String, List<String>>();
+          while (service.getRemainingCount() > 0) {
+            Future<Map<String, List<String>>> future = service.poll(_defaultParallelCallTimeout,
TimeUnit.MILLISECONDS, true);
+            Map<String, List<String>> snapshotsOnAShardServer = service.getResultThrowException(future);
+            for (Entry<String, List<String>> entry : snapshotsOnAShardServer.entrySet())
{
+              List<String> snapshots = result.get(entry.getKey());
+              if (snapshots == null) {
+                snapshots = new ArrayList<String>();
+                result.put(entry.getKey(), snapshots);
+              }
+              snapshots.addAll(entry.getValue());
+            }
+          }
+          return result;
+        }
+      });
+    } catch (Exception e) {
+      LOG.error("Unknown error while trying to get the list of snapshots for table [{0}]",
e, table);
+      throw new BException("Unknown error while trying to get the list of snapshots for table
[{0}]", e, table);
+    }
+  }
 
   public void setNodeName(String nodeName) {
     _nodeName = nodeName;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
index edd3d80..61cf775 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
@@ -21,12 +21,14 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIV
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLongArray;
 
@@ -41,6 +43,8 @@ import org.apache.blur.manager.results.BlurResultIterable;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.server.ShardServerContext;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.commands.BlurCommand;
+import org.apache.blur.thrift.generated.Blur.Client;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
@@ -56,10 +60,12 @@ import org.apache.blur.thrift.generated.SimpleQuery;
 import org.apache.blur.thrift.generated.Status;
 import org.apache.blur.thrift.generated.TableStats;
 import org.apache.blur.utils.BlurConstants;
+import org.apache.blur.utils.BlurExecutorCompletionService;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.QueryCache;
 import org.apache.blur.utils.QueryCacheEntry;
 import org.apache.blur.utils.QueryCacheKey;
+import org.apache.blur.utils.ForkJoin.Merger;
 
 public class BlurShardServer extends TableAdmin implements Iface {
 
@@ -397,6 +403,63 @@ public class BlurShardServer extends TableAdmin implements Iface {
       throw new BException(e.getMessage(), e);
     }
   }
+  
+  @Override
+  public void createSnapshot(final String table, final String name) throws BlurException,
TException {
+    try {
+      Map<String, BlurIndex> indexes = _indexServer.getIndexes(table);
+      for (Entry<String, BlurIndex> entry : indexes.entrySet()) {
+        BlurIndex index = entry.getValue();
+        index.createSnapshot(name);
+      }
+    } catch (Exception e) {
+      LOG.error("Unknown error while trying to getting indexes for table [" + table + "]",
e);
+      if (e instanceof BlurException) {
+        throw (BlurException) e;
+      }
+      throw new BException(e.getMessage(), e);
+    }
+  }
+  
+  @Override
+  public void removeSnapshot(final String table, final String name) throws BlurException,
TException {
+    try {
+      Map<String, BlurIndex> indexes = _indexServer.getIndexes(table);
+      for (Entry<String, BlurIndex> entry : indexes.entrySet()) {
+        BlurIndex index = entry.getValue();
+        index.removeSnapshot(name);
+      }
+    } catch (Exception e) {
+      LOG.error("Unknown error while trying to getting indexes for table [" + table + "]",
e);
+      if (e instanceof BlurException) {
+        throw (BlurException) e;
+      }
+      throw new BException(e.getMessage(), e);
+    }
+  }
+  
+  @Override
+  public Map<String, List<String>> listSnapshots(final String table) throws BlurException,
TException {
+    Map<String, List<String>> snapshots = new HashMap<String, List<String>>();
+    try {
+      Map<String, BlurIndex> indexes = _indexServer.getIndexes(table);
+      for (Entry<String, BlurIndex> entry : indexes.entrySet()) {
+        BlurIndex index = entry.getValue();
+        List<String> shardSnapshots = index.getSnapshots();
+        if (shardSnapshots == null) {
+          shardSnapshots = new ArrayList<String>();
+        }
+        snapshots.put(entry.getKey(), shardSnapshots);
+      }
+      return snapshots;
+    } catch (Exception e) {
+      LOG.error("Unknown error while trying to getting indexes for table [" + table + "]",
e);
+      if (e instanceof BlurException) {
+        throw (BlurException) e;
+      }
+      throw new BException(e.getMessage(), e);
+    }
+  }
 
   public int getDataFetchThreadCount() {
     return _dataFetchThreadCount;


Mime
View raw message