incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Dealing with differences between a local file system and a real hdfs file system.
Date Wed, 18 Mar 2015 13:40:40 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 6d3823712 -> f02bd95a0


Dealing with differences between a local file system and a real hdfs file system.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/f02bd95a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/f02bd95a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/f02bd95a

Branch: refs/heads/master
Commit: f02bd95a025d9dfb43728e02dd55639a745abef9
Parents: 6d38237
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Mar 18 09:40:41 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Mar 18 09:40:41 2015 -0400

----------------------------------------------------------------------
 .../org/apache/blur/store/hdfs/HdfsDirectory.java    | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f02bd95a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
index a727768..9396638 100644
--- a/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
+++ b/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
@@ -132,11 +132,16 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
   protected final Map<String, Path> _symlinkPathMap = new ConcurrentHashMap<String,
Path>();
   protected final Map<Path, FSDataInputStream> _inputMap = new ConcurrentHashMap<Path,
FSDataInputStream>();
   protected final boolean _useCache = true;
+  protected final boolean _asyncClosing;
 
   public HdfsDirectory(Configuration configuration, Path path) throws IOException {
     _fileSystem = path.getFileSystem(configuration);
-
     _path = _fileSystem.makeQualified(path);
+    if (_path.toUri().getScheme().equals("hdfs")) {
+      _asyncClosing = true;
+    } else {
+      _asyncClosing = false;
+    }
     _fileSystem.mkdirs(path);
     setLockFactory(NoLockFactory.getNoLockFactory());
     synchronized (_metricsGroupMap) {
@@ -260,9 +265,13 @@ public class HdfsDirectory extends Directory implements LastModified,
HdfsSymlin
       @Override
       public void close() throws IOException {
         super.close();
-        outputStream.sync();
         _fileStatusMap.put(name, new FStat(System.currentTimeMillis(), outputStream.getPos()));
-        CLOSING_QUEUE.add(outputStream);
+        if (_asyncClosing) {
+          outputStream.sync();
+          CLOSING_QUEUE.add(outputStream);
+        } else {
+          outputStream.close();
+        }
         openForInput(name);
       }
 


Mime
View raw message