incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Handling edge case where a index import is committed into the main index but the inprogress file is not removed before operation completes.
Date Thu, 16 Apr 2015 01:07:35 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 26eff4e0e -> 4aa976fd0


Handling edge case where a index import is committed into the main index but the inprogress
file is not removed before operation completes.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/4aa976fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/4aa976fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/4aa976fd

Branch: refs/heads/master
Commit: 4aa976fd064076858944f0f0e4b9d21ff52f21a5
Parents: 26eff4e
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Apr 15 21:07:44 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Apr 15 21:07:44 2015 -0400

----------------------------------------------------------------------
 .../blur/manager/writer/BlurIndexSimpleWriter.java    |  2 +-
 .../org/apache/blur/manager/writer/IndexImporter.java | 14 +++++++++++++-
 .../apache/blur/manager/writer/IndexImporterTest.java |  2 +-
 3 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4aa976fd/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
index 5e59c01..fbec370 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java
@@ -287,7 +287,7 @@ public class BlurIndexSimpleWriter extends BlurIndex {
             _writer.notify();
           }
           _indexImporter = new IndexImporter(_indexImporterTimer, BlurIndexSimpleWriter.this,
_shardContext,
-              TimeUnit.SECONDS, 10);
+              TimeUnit.SECONDS, 10, _thriftCache);
         } catch (IOException e) {
           LOG.error("Unknown error on index writer open.", e);
         }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4aa976fd/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
index 99c4f71..d45876e 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
@@ -38,6 +38,7 @@ import org.apache.blur.lucene.search.IndexSearcherCloseable;
 import org.apache.blur.manager.BlurPartitioner;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
+import org.apache.blur.server.cache.ThriftCache;
 import org.apache.blur.store.hdfs.HdfsDirectory;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.ShardUtil;
@@ -76,12 +77,14 @@ public class IndexImporter extends TimerTask implements Closeable {
   private final String _shard;
   private final long _cleanupDelay;
   private final Timer _inindexImporterTimer;
+  private final ThriftCache _thriftCache;
 
   private long _lastCleanup;
   private Runnable _testError;
 
   public IndexImporter(Timer indexImporterTimer, BlurIndex blurIndex, ShardContext shardContext,
TimeUnit refreshUnit,
-      long refreshAmount) {
+      long refreshAmount, ThriftCache thriftCache) {
+    _thriftCache = thriftCache;
     _blurIndex = blurIndex;
     _shardContext = shardContext;
 
@@ -362,6 +365,15 @@ public class IndexImporter extends TimerTask implements Closeable {
       Path realPath = HdfsDirectory.readRealPathDataFromSymlinkPath(fileSystem, status.getPath());
       Path inuseDir = inuseFileToDir.get(realPath);
       inuseDirs.remove(inuseDir);
+      // if the inuse dir has an inprogress file then remove it because there
+      // are files that reference this dir so it had to be committed.
+      Path path = new Path(inuseDir, INPROGRESS);
+      if (fileSystem.exists(path)) {
+        fileSystem.delete(path, false);
+        if (_thriftCache != null) {
+          _thriftCache.clearTable(_table);
+        }
+      }
     }
 
     // Check if any inuse dirs have inprogress files.

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4aa976fd/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
index 82772b6..b791c4a 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
@@ -122,7 +122,7 @@ public class IndexImporterTest {
     BufferStore.initNewBuffer(128, 128 * 128);
 
     _indexImporter = new IndexImporter(_timer, getBlurIndex(shardContext, _mainDirectory),
shardContext,
-        TimeUnit.MINUTES, 10);
+        TimeUnit.MINUTES, 10, null);
   }
 
   private BlurIndex getBlurIndex(ShardContext shardContext, final Directory mainDirectory)
throws IOException {


Mime
View raw message