incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [9/9] git commit: Added some counters to aid in creating some in directory caches for performance speed up.
Date Tue, 19 Feb 2013 01:59:11 GMT
Updated Branches:
  refs/heads/0.2-dev 747caacfd -> 203b8bd55


Added some counters to aid in creating some in directory caches for performance speed up.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/203b8bd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/203b8bd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/203b8bd5

Branch: refs/heads/0.2-dev
Commit: 203b8bd55d88b1c2b4eaca29944668b6a6940f9e
Parents: a5a38ff
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Feb 18 20:57:22 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Feb 18 20:57:22 2013 -0500

----------------------------------------------------------------------
 .../org/apache/blur/store/hdfs/HdfsDirectory.java  |   46 +++++++++++++++
 1 files changed, 46 insertions(+), 0 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/203b8bd5/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
index f7c9d53..366e1c7 100644
--- a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
+++ b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
@@ -20,7 +20,10 @@ package org.apache.blur.store.hdfs;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
 import org.apache.blur.store.blockcache.BlockDirectory;
 import org.apache.blur.store.buffer.ReusedBufferedIndexInput;
 import org.apache.hadoop.conf.Configuration;
@@ -39,6 +42,41 @@ import org.apache.lucene.store.NoLockFactory;
 
 public class HdfsDirectory extends Directory {
 
+  private static final Log LOG = LogFactory.getLog(HdfsDirectory.class);
+
+  private static AtomicLong deleteCounter = new AtomicLong();
+  private static AtomicLong existsCounter = new AtomicLong();
+  private static AtomicLong fileStatusCounter = new AtomicLong();
+  private static AtomicLong renameCounter = new AtomicLong();
+  private static AtomicLong listCounter = new AtomicLong();
+  private static AtomicLong createCounter = new AtomicLong();
+  private static AtomicLong isFileCounter = new AtomicLong();
+
+  static {
+    Thread thread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        while (true) {
+          LOG.info("Delete Counter [" + deleteCounter + "]");
+          LOG.info("Exists Counter [" + existsCounter + "]");
+          LOG.info("File Status Counter [" + fileStatusCounter + "]");
+          LOG.info("Rename Counter [" + renameCounter + "]");
+          LOG.info("List Counter [" + listCounter + "]");
+          LOG.info("Create Counter [" + createCounter + "]");
+          LOG.info("IsFile Counter [" + isFileCounter + "]");
+          try {
+            Thread.sleep(5000);
+          } catch (InterruptedException e) {
+            return;
+          }
+        }
+      }
+    });
+    thread.setName("HDFS dir counter logger");
+    thread.setDaemon(true);
+    thread.start();
+  }
+
   private final Path path;
   private final FileSystem fileSystem;
 
@@ -97,6 +135,7 @@ public class HdfsDirectory extends Directory {
       throw new IOException("File [" + name + "] already exists found.");
     }
     final FSDataOutputStream outputStream = fileSystem.create(getPath(name));
+    createCounter.incrementAndGet();
     return new BufferedIndexOutput() {
 
       @Override
@@ -133,10 +172,12 @@ public class HdfsDirectory extends Directory {
 
   @Override
   public String[] listAll() throws IOException {
+    listCounter.incrementAndGet();
     FileStatus[] files = fileSystem.listStatus(path, new PathFilter() {
       @Override
       public boolean accept(Path path) {
         try {
+          isFileCounter.incrementAndGet();
           return fileSystem.isFile(path);
         } catch (IOException e) {
           throw new RuntimeException(e);
@@ -152,12 +193,14 @@ public class HdfsDirectory extends Directory {
 
   @Override
   public boolean fileExists(String name) throws IOException {
+    existsCounter.incrementAndGet();
     return fileSystem.exists(getPath(name));
   }
 
   @Override
   public void deleteFile(String name) throws IOException {
     if (fileExists(name)) {
+      deleteCounter.incrementAndGet();
       fileSystem.delete(getPath(name), true);
     } else {
       throw new FileNotFoundException("File [" + name + "] not found");
@@ -166,6 +209,7 @@ public class HdfsDirectory extends Directory {
 
   @Override
   public long fileLength(String name) throws IOException {
+    fileStatusCounter.incrementAndGet();
     FileStatus fileStatus = fileSystem.getFileStatus(getPath(name));
     return fileStatus.getLen();
   }
@@ -189,6 +233,7 @@ public class HdfsDirectory extends Directory {
       throw new FileNotFoundException("File [" + name + "] not found");
     }
     Path file = getPath(name);
+    fileStatusCounter.incrementAndGet();
     return fileSystem.getFileStatus(file).getModificationTime();
   }
 
@@ -213,6 +258,7 @@ public class HdfsDirectory extends Directory {
     if (ifSameCluster(to, this)) {
       Path newDest = simpleTo.getPath(dest);
       Path oldSrc = getPath(src);
+      renameCounter.incrementAndGet();
       return fileSystem.rename(oldSrc, newDest);
     }
     return false;


Mime
View raw message