incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/3] git commit: Adding last mod time to internal hdfs directory cache.
Date Tue, 28 Oct 2014 13:26:51 GMT
Adding last mod time to internal hdfs directory cache.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2f3c8103
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2f3c8103
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2f3c8103

Branch: refs/heads/master
Commit: 2f3c8103000c961f845f64d074cc86e2780c26ed
Parents: 1fac39c
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Oct 28 09:26:26 2014 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Oct 28 09:26:26 2014 -0400

----------------------------------------------------------------------
 .../apache/blur/store/hdfs/HdfsDirectory.java   | 52 ++++++++++++++------
 1 file changed, 37 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2f3c8103/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
index bd1a236..e770313 100644
--- a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
@@ -69,10 +69,24 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
    */
   protected static Map<URI, MetricsGroup> _metricsGroupMap = new WeakHashMap<URI,
MetricsGroup>();
 
+  static class FStat {
+    FStat(FileStatus fileStatus) {
+      this(fileStatus.getModificationTime(), fileStatus.getLen());
+    }
+
+    FStat(long lastMod, long length) {
+      _lastMod = lastMod;
+      _length = length;
+    }
+
+    final long _lastMod;
+    final long _length;
+  }
+
   protected final Path _path;
   protected final FileSystem _fileSystem;
   protected final MetricsGroup _metricsGroup;
-  protected final Map<String, Long> _fileLengthMap = new ConcurrentHashMap<String,
Long>();
+  protected final Map<String, FStat> _fileStatusMap = new ConcurrentHashMap<String,
FStat>();
   protected final Map<String, Boolean> _symlinkMap = new ConcurrentHashMap<String,
Boolean>();
   protected final Map<String, Path> _symlinkPathMap = new ConcurrentHashMap<String,
Path>();
   protected final Map<Path, FSDataInputStream> _inputMap = new ConcurrentHashMap<Path,
FSDataInputStream>();
@@ -103,9 +117,9 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
             String resolvedName = getRealFileName(name);
             Path resolvedPath = getPath(resolvedName);
             FileStatus resolvedFileStatus = _fileSystem.getFileStatus(resolvedPath);
-            _fileLengthMap.put(resolvedName, resolvedFileStatus.getLen());
+            _fileStatusMap.put(resolvedName, new FStat(resolvedFileStatus));
           } else {
-            _fileLengthMap.put(name, fileStatus.getLen());
+            _fileStatusMap.put(name, new FStat(fileStatus));
           }
         }
       }
@@ -144,7 +158,7 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
     if (fileExists(name)) {
       throw new IOException("File [" + name + "] already exists found.");
     }
-    _fileLengthMap.put(name, 0L);
+    _fileStatusMap.put(name, new FStat(System.currentTimeMillis(), 0L));
     final FSDataOutputStream outputStream = openForOutput(name);
     return new BufferedIndexOutput() {
 
@@ -165,7 +179,7 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
       @Override
       public void close() throws IOException {
         super.close();
-        _fileLengthMap.put(name, outputStream.getPos());
+        _fileStatusMap.put(name, new FStat(System.currentTimeMillis(), outputStream.getPos()));
         outputStream.close();
         openForInput(name);
       }
@@ -219,7 +233,7 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
     LOG.debug("listAll [{0}]", _path);
 
     if (_useCache) {
-      Set<String> names = new HashSet<String>(_fileLengthMap.keySet());
+      Set<String> names = new HashSet<String>(_fileStatusMap.keySet());
       return names.toArray(new String[names.size()]);
     }
 
@@ -249,7 +263,7 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
   public boolean fileExists(String name) throws IOException {
     LOG.debug("fileExists [{0}] [{1}]", name, _path);
     if (_useCache) {
-      return _fileLengthMap.containsKey(name);
+      return _fileStatusMap.containsKey(name);
     }
     return exists(name);
   }
@@ -269,7 +283,7 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
     LOG.debug("deleteFile [{0}] [{1}]", name, _path);
     if (fileExists(name)) {
       if (_useCache) {
-        _fileLengthMap.remove(name);
+        _fileStatusMap.remove(name);
       }
       delete(name);
     } else {
@@ -299,11 +313,11 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
   public long fileLength(String name) throws IOException {
     LOG.debug("fileLength [{0}] [{1}]", name, _path);
     if (_useCache) {
-      Long length = _fileLengthMap.get(name);
-      if (length == null) {
+      FStat fStat = _fileStatusMap.get(name);
+      if (fStat == null) {
         throw new FileNotFoundException(name);
       }
-      return length;
+      return fStat._length;
     }
 
     return length(name);
@@ -399,8 +413,12 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
   }
 
   public long getFileModified(String name) throws IOException {
-    if (!fileExists(name)) {
-      throw new FileNotFoundException("File [" + name + "] not found");
+    if (_useCache) {
+      FStat fStat = _fileStatusMap.get(name);
+      if (fStat == null) {
+        throw new FileNotFoundException("File [" + name + "] not found");
+      }
+      return fStat._lastMod;
     }
     return fileModified(name);
   }
@@ -409,7 +427,11 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
     Path path = getPath(name);
     Tracer trace = Trace.trace("filesystem - fileModified", Trace.param("path", path));
     try {
-      return _fileSystem.getFileStatus(path).getModificationTime();
+      FileStatus fileStatus = _fileSystem.getFileStatus(path);
+      if (_useCache) {
+        _fileStatusMap.put(name, new FStat(fileStatus));
+      }
+      return fileStatus.getModificationTime();
     } finally {
       trace.done();
     }
@@ -439,7 +461,7 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
     outputStream.write(srcPath.toString().getBytes(UTF_8));
     outputStream.close();
     if (_useCache) {
-      to._fileLengthMap.put(dest, _fileLengthMap.get(src));
+      to._fileStatusMap.put(dest, _fileStatusMap.get(src));
     }
     return true;
   }


Mime
View raw message