incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [5/7] git commit: More fixes to the WAL and some check for testing to not throw a bunch of exceptions.
Date Wed, 22 May 2013 23:46:33 GMT
More fixes to the WAL and some check for testing to not throw a bunch of exceptions.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/c17a39a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/c17a39a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/c17a39a3

Branch: refs/heads/0.1.5
Commit: c17a39a323da8ec7b8cc53d7130c41133aa12c94
Parents: b4852bd
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed May 22 10:38:35 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed May 22 10:38:35 2013 -0400

----------------------------------------------------------------------
 .../apache/blur/manager/writer/BlurNRTIndex.java   |   11 +++++++++
 .../blur/manager/writer/TransactionRecorder.java   |   14 ++++++++++-
 .../org/apache/blur/thrift/BlurClusterTest.java    |    5 ----
 .../java/org/apache/blur/index/IndexWriter.java    |   18 +++++++++++---
 4 files changed, 37 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c17a39a3/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
b/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index c10c713..9adbc46 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -28,6 +28,7 @@ import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.blur.index.IndexWriter;
+import org.apache.blur.index.IndexWriter.LockOwnerException;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter;
@@ -219,6 +220,16 @@ public class BlurNRTIndex extends BlurIndex {
           } catch (CorruptIndexException e) {
             LOG.error("Curruption Error during commit of [{0}/{1}].", e, _tableContext.getTable(),
                 _shardContext.getShard());
+          } catch (LockOwnerException e) {
+            LOG.info("This shard server no longer owns the lock on [{0}/{1}], closing.",
_tableContext.getTable(),
+                _shardContext.getShard());
+            try {
+              close();
+            } catch (IOException ex) {
+              LOG.error("Unknown error while trying to close [{0}/{1}]", _tableContext.getTable(),
+                  _shardContext.getShard());
+            }
+            return;
           } catch (IOException e) {
             LOG.error("IO Error during commit of [{0}/{1}].", e, _tableContext.getTable(),
_shardContext.getShard());
           }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c17a39a3/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
b/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
index 421d4cb..a728290 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
@@ -18,6 +18,7 @@ package org.apache.blur.manager.writer;
  */
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.EOFException;
@@ -57,7 +58,7 @@ import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.NRTManager.TrackingIndexWriter;
 
-public class TransactionRecorder extends TimerTask {
+public class TransactionRecorder extends TimerTask implements Closeable {
 
   enum TYPE {
     DELETE((byte) 0), ROW((byte) 1);
@@ -385,7 +386,16 @@ public class TransactionRecorder extends TimerTask {
       }
     } catch (IOException e) {
       if (_running.get()) {
-        LOG.error("Known error while trying to sync.", e);
+        if (e.getMessage().equals("DFSOutputStream is closed")) {
+          LOG.warn("Trying to sync the outputstrema and the stream has been closed.  This
is probably a test and the filesystem has been closed.");
+          try {
+            Thread.sleep(TimeUnit.SECONDS.toMillis(5));
+          } catch (InterruptedException ex) {
+            return;
+          }
+        } else {
+          LOG.error("Known error while trying to sync.", e);
+        }
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c17a39a3/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
index e1b6cd0..504f26a 100644
--- a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java
@@ -143,11 +143,6 @@ public class BlurClusterTest {
     //This should block until shards have failed over
     client.shardServerLayout("test");
     
-    for (int i = 0; i < 40; i++) {
-      System.out.println(client.query("test", blurQuery).getTotalResults());
-      Thread.sleep(1000);
-    }
-    
     assertEquals(length, client.query("test", blurQuery).getTotalResults());
     
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c17a39a3/src/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java b/src/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java
index 4080932..95f8b05 100644
--- a/src/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java
+++ b/src/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java
@@ -27,9 +27,18 @@ import org.apache.lucene.store.LockObtainFailedException;
 
 public class IndexWriter extends org.apache.lucene.index.IndexWriter {
 
+  public static class LockOwnerException extends IOException {
+    private static final long serialVersionUID = -8211546713487754992L;
+
+    public LockOwnerException(String msg) {
+      super(msg);
+    }
+  }
+
   private Lock internalLock;
 
-  public IndexWriter(Directory d, IndexWriterConfig conf) throws CorruptIndexException, LockObtainFailedException,
IOException {
+  public IndexWriter(Directory d, IndexWriterConfig conf) throws CorruptIndexException, LockObtainFailedException,
+      IOException {
     super(d, conf);
     try {
       internalLock = getInternalLock();
@@ -38,7 +47,8 @@ public class IndexWriter extends org.apache.lucene.index.IndexWriter {
     }
   }
 
-  private Lock getInternalLock() throws SecurityException, NoSuchFieldException, IllegalArgumentException,
IllegalAccessException {
+  private Lock getInternalLock() throws SecurityException, NoSuchFieldException, IllegalArgumentException,
+      IllegalAccessException {
     Field field = org.apache.lucene.index.IndexWriter.class.getDeclaredField("writeLock");
     field.setAccessible(true);
     return (Lock) field.get(this);
@@ -48,7 +58,7 @@ public class IndexWriter extends org.apache.lucene.index.IndexWriter {
   protected void doAfterFlush() throws IOException {
     super.doAfterFlush();
     if (!internalLock.isLocked()) {
-      throw new IOException("Lock [" + internalLock + "] no longer has write lock.");
+      throw new LockOwnerException("Lock [" + internalLock + "] no longer has write lock.");
     }
   }
 
@@ -56,7 +66,7 @@ public class IndexWriter extends org.apache.lucene.index.IndexWriter {
   protected void doBeforeFlush() throws IOException {
     super.doBeforeFlush();
     if (!internalLock.isLocked()) {
-      throw new IOException("Lock [" + internalLock + "] no longer has write lock.");
+      throw new LockOwnerException("Lock [" + internalLock + "] no longer has write lock.");
     }
   }
 


Mime
View raw message