incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/3] git commit: Fixing a bug where the last modified time of the file was not taken into accoount. Therefore if a table was destroyed and recreated the files present in the cache could have been used into of the real files in the base directory.
Date Sat, 08 Feb 2014 22:55:46 GMT
Updated Branches:
  refs/heads/apache-blur-0.2 103ac9785 -> 0756d357d


Fixing a bug where the last modified time of the file was not taken into accoount.  Therefore
if a table was destroyed and recreated the files present in the cache could have been used
into of the real files in the base directory.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ddf48ff6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ddf48ff6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ddf48ff6

Branch: refs/heads/apache-blur-0.2
Commit: ddf48ff6b465a8eaf02022c8e28a35ae8e73ac61
Parents: 103ac97
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Feb 8 14:20:49 2014 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Feb 8 14:20:49 2014 -0500

----------------------------------------------------------------------
 .../blur/store/blockcache_v2/BaseCache.java     | 27 ++++++++++++++++++--
 .../apache/blur/store/blockcache_v2/Cache.java  | 13 ++++++++++
 .../store/blockcache_v2/CacheIndexOutput.java   |  1 +
 .../blur/store/hdfs_v2/JoinDirectory.java       |  9 +++----
 .../blockcache_v2/CacheIndexInputTest.java      |  5 ++++
 5 files changed, 48 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
index bcb279c..796f59d 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/BaseCache.java
@@ -27,6 +27,7 @@ import static org.apache.blur.metrics.MetricsConstants.REMOVAL;
 import static org.apache.blur.metrics.MetricsConstants.SIZE;
 
 import java.io.Closeable;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -193,9 +194,31 @@ public class BaseCache extends Cache implements Closeable {
     _fileNameToId.remove(cachedFileName);
   }
 
+  @Override
+  public void fileClosedForWriting(CacheDirectory directory, String fileName, long fileId)
throws IOException {
+    if (directory.fileExists(fileName)) {
+      long fileModified = directory.getFileModified(fileName);
+      FileIdKey oldKey = new FileIdKey(directory.getDirectoryName(), fileName, -1L);
+      FileIdKey newKey = new FileIdKey(directory.getDirectoryName(), fileName, fileModified);
+      long currentFileId = _fileNameToId.get(oldKey);
+      if (fileId != currentFileId) {
+        throw new IOException("Something has gone very wrong file ids do not match [" + fileId
+ "] [" + currentFileId
+            + "] for key [" + oldKey + "]");
+      }
+      _fileNameToId.put(newKey, fileId);
+      _oldFileNameIdMap.put(fileId, newKey);
+      _fileNameToId.remove(oldKey);
+    } else {
+      throw new FileNotFoundException("File [" + fileName + "] not found in directory ["
+ directory + "]");
+    }
+  }
+
   private FileIdKey getCacheFileName(CacheDirectory directory, String fileName) throws IOException
{
-    long fileModified = directory.getFileModified(fileName);
-    return new FileIdKey(directory.getDirectoryName(), fileName, fileModified);
+    if (directory.fileExists(fileName)) {
+      long fileModified = directory.getFileModified(fileName);
+      return new FileIdKey(directory.getDirectoryName(), fileName, fileModified);
+    }
+    return new FileIdKey(directory.getDirectoryName(), fileName, -1L);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java
index 1dddd75..402c55c 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/Cache.java
@@ -173,4 +173,17 @@ public abstract class Cache implements Closeable {
    */
   public abstract boolean shouldBeQuiet(CacheDirectory directory, String fileName);
 
+  /**
+   * The cache internals rely on the last modified timestamp of a given file to
+   * know if the file is the same or not. During the writing of a given file the
+   * last moified date is not know, so this method tells the cache that the file
+   * has completed the writing phase and the last modified time should now be
+   * accurate.
+   * 
+   * @param fileId
+   *          the file id.
+   * @throws IOException 
+   */
+  public abstract void fileClosedForWriting(CacheDirectory directory, String fileName, long
fileId) throws IOException;
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
index dd0dc29..caad489 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/CacheIndexOutput.java
@@ -129,6 +129,7 @@ public class CacheIndexOutput extends IndexOutput {
     _indexOutput.flush();
     _indexOutput.close();
     _store.putBuffer(_buffer);
+    _cache.fileClosedForWriting(_directory, _fileName, _fileId);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java
index 236c7e5..3fba841 100644
--- a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java
@@ -150,11 +150,10 @@ public class JoinDirectory extends Directory implements LastModified
{
 
   @Override
   public long getFileModified(String name) throws IOException {
-    return 0;
-    // if (_shortTermStorage.fileExists(name)) {
-    // return ((LastModified) _shortTermStorage).getFileModified(name);
-    // }
-    // return ((LastModified) _longTermStorage).getFileModified(name);
+    if (_shortTermStorage.fileExists(name)) {
+      return ((LastModified) _shortTermStorage).getFileModified(name);
+    }
+    return ((LastModified) _longTermStorage).getFileModified(name);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ddf48ff6/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
index ee4b1bb..36adfcc 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache_v2/CacheIndexInputTest.java
@@ -256,6 +256,11 @@ public class CacheIndexInputTest {
 
       }
 
+      @Override
+      public void fileClosedForWriting(CacheDirectory directory, String fileName, long fileId)
throws IOException {
+        
+      }
+
     };
     return cacheFactory;
   }


Mime
View raw message