incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Adding new metrics library, plus some new metrics.
Date Wed, 06 Mar 2013 04:32:30 GMT
Updated Branches:
  refs/heads/0.2-dev 1a8ebf48c -> 0270f9a09


Adding new metrics library, plus some new metrics.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0270f9a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0270f9a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0270f9a0

Branch: refs/heads/0.2-dev
Commit: 0270f9a09f314e51d6b2df5d69f4dab10512b9e5
Parents: 1a8ebf4
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Mar 5 23:30:40 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Mar 5 23:30:40 2013 -0500

----------------------------------------------------------------------
 .../java/org/apache/blur/server/BlurServer.java    |   30 ++++++++++++
 .../store/refcounter/DirectoryReferenceFileGC.java |   14 ++++++
 .../org/apache/blur/store/hdfs/HdfsDirectory.java  |   26 ++++++++++-
 src/blur-util/pom.xml                              |   36 +++++++--------
 src/pom.xml                                        |    1 +
 5 files changed, 86 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java b/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java
index e6185ea..9dfda86 100644
--- a/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/server/BlurServer.java
@@ -74,6 +74,9 @@ import org.apache.blur.utils.ThriftLuceneConversion;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.thrift.TException;
 
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Meter;
+
 public class BlurServer extends TableAdmin implements Iface {
 
   private static final Log LOG = LogFactory.getLog(BlurServer.class);
@@ -88,6 +91,14 @@ public class BlurServer extends TableAdmin implements Iface {
   private SessionManager _sessionManager;
   private String _nodeName;
   private QueryStatusContainer _queryStatusContainer;
+  private Meter localSearches;
+  private Meter remoteSearches;
+  private Meter localAddRequests;
+  private Meter remoteAddRequests;
+  private Meter localUpdateRequests;
+  private Meter remoteUpdateRequests;
+  private Meter localDeleteRequests;
+  private Meter remoteDeleteRequests;
 
   static class SearchAction {
     enum ACTION {
@@ -126,6 +137,15 @@ public class BlurServer extends TableAdmin implements Iface {
     _queryStatusContainer = new QueryStatusContainer(_configuration.getInt(BLUR_QUERYSTATUS_TRACKING_MAX,
16384));
     _sessionManager = new SessionManager();
     _sessionManager.init();
+    
+    localSearches = Metrics.newMeter(BlurServer.class, "searches", "local", TimeUnit.SECONDS);
+    remoteSearches = Metrics.newMeter(BlurServer.class, "searches", "remote", TimeUnit.SECONDS);
+    localAddRequests = Metrics.newMeter(BlurServer.class, "addRequests", "local", TimeUnit.SECONDS);
+    remoteAddRequests = Metrics.newMeter(BlurServer.class, "addRequests", "remote", TimeUnit.SECONDS);
+    localUpdateRequests = Metrics.newMeter(BlurServer.class, "updateRequests", "local", TimeUnit.SECONDS);
+    remoteUpdateRequests = Metrics.newMeter(BlurServer.class, "updateRequests", "remote",
TimeUnit.SECONDS);
+    localDeleteRequests = Metrics.newMeter(BlurServer.class, "deleteRequests", "local", TimeUnit.SECONDS);
+    remoteDeleteRequests = Metrics.newMeter(BlurServer.class, "deleteRequests", "remote",
TimeUnit.SECONDS);
   }
 
   @Override
@@ -219,11 +239,13 @@ public class BlurServer extends TableAdmin implements Iface {
         final int shardIndex = action.shardIndex;
         if (action.type == ACTION.LOCAL) {
           searchCallable = new LocalSearchCallable(shardIndex, action.indexSearcher, after,
query, filter, sort, numberToFetch, doDocScores, doMaxScore);
+          localSearches.mark();
           future = _indexSearcherExecutor.submit(searchCallable);
         } else if (action.type == ACTION.REMOTE) {
           // @TODO need to send only one call per server, instead of one for
           // each shard in the table
           searchCallable = new RemoteSearchCallable(session, action.remoteServer, shardIndex,
queryArgs);
+          remoteSearches.mark();
           future = _indexSearcherExecutor.submit(searchCallable);
         } else {
           throw new BlurException("Unsupported action of [" + action.type + "]", null);
@@ -343,8 +365,10 @@ public class BlurServer extends TableAdmin implements Iface {
       BlurIndex index = getIndex(table, shardIndex);
       List<Document> indexableDocuments = BlurValidations.getAllIndexableDocuments(documents);
       if (index == null) {
+        remoteAddRequests.mark(documents.size());
         generations.addAll(forwardAddDocuments(options, indexableDocuments));
       } else {
+        localAddRequests.mark(documents.size());
         long generation = index.addDocuments(waitToBeVisible, writeAheadLog, indexableDocuments);
         generations.add(new Generation(table, shardIndex, generation));
       }
@@ -404,8 +428,10 @@ public class BlurServer extends TableAdmin implements Iface {
     try {
       BlurIndex index = getIndex(table, shardIndex);
       if (index == null) {
+        remoteDeleteRequests.mark(queries.size());
         generations.addAll(forwardDeleteDocumentsByQueries(options, queries));
       } else {
+        localDeleteRequests.mark(queries.size());
         long generation = index.deleteDocuments(waitToBeVisible, writeAheadLog, queries.toArray(new
Query[queries.size()]));
         generations.add(new Generation(table, shardIndex, generation));
       }
@@ -426,8 +452,10 @@ public class BlurServer extends TableAdmin implements Iface {
     try {
       BlurIndex index = getIndex(table, shardIndex);
       if (index == null) {
+        remoteDeleteRequests.mark(terms.size());
         generations.addAll(forwardDeleteDocuments(options, terms));
       } else {
+        localDeleteRequests.mark(terms.size());
         long generation = index.deleteDocuments(waitToBeVisible, writeAheadLog, terms.toArray(new
Term[terms.size()]));
         generations.add(new Generation(table, shardIndex, generation));
       }
@@ -449,8 +477,10 @@ public class BlurServer extends TableAdmin implements Iface {
       BlurIndex index = getIndex(table, shardIndex);
       List<UpdatePackage> updatePackagesWithIndexableDocs = BlurValidations.getAllIndexablePackages(updatePackages);
       if (index == null) {
+        remoteUpdateRequests.mark(updatePackages.size());
         generations.addAll(forwardUpdateDocuments(options, updatePackagesWithIndexableDocs));
       } else {
+        localUpdateRequests.mark(updatePackages.size());
         long generation = index.updateDocuments(waitToBeVisible, writeAheadLog, updatePackagesWithIndexableDocs);
         generations.add(new Generation(table, shardIndex, generation));
       }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java
b/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java
index 9db6256..b8a9301 100644
--- a/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java
+++ b/src/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java
@@ -28,6 +28,9 @@ import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.lucene.store.Directory;
 
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Gauge;
+
 public class DirectoryReferenceFileGC extends TimerTask {
 
   private static final Log LOG = LogFactory.getLog(DirectoryReferenceFileGC.class);
@@ -35,6 +38,7 @@ public class DirectoryReferenceFileGC extends TimerTask {
   private Timer _timer;
   private long _delay = 5000;
   private LinkedBlockingQueue<Value> _queue;
+  private volatile int numberOfFilesToBeDeleted = 0;
 
   public static class Value {
     public Value(Directory directory, String name, Map<String, AtomicInteger> refs)
{
@@ -65,6 +69,12 @@ public class DirectoryReferenceFileGC extends TimerTask {
     _timer = new Timer("Blur-File-GC", true);
     _timer.scheduleAtFixedRate(this, _delay, _delay);
     _queue = new LinkedBlockingQueue<Value>();
+    Metrics.newGauge(getClass(), "filesToBeDeleted", new Gauge<Integer>() {
+      @Override
+      public Integer value() {
+        return numberOfFilesToBeDeleted;
+      }
+    });
   }
 
   public void add(Directory directory, String name, Map<String, AtomicInteger> refs)
{
@@ -83,15 +93,19 @@ public class DirectoryReferenceFileGC extends TimerTask {
   @Override
   public void run() {
     Iterator<Value> iterator = _queue.iterator();
+    int count = 0;
     while (iterator.hasNext()) {
       Value value = iterator.next();
       try {
         if (value.tryToDelete()) {
           iterator.remove();
+        } else {
+          count++;
         }
       } catch (IOException e) {
         LOG.error("Unknown error", e);
       }
     }
+    numberOfFilesToBeDeleted = count;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
index 18de3e7..fdc5a52 100644
--- a/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
+++ b/src/blur-store/src/main/java/org/apache/blur/store/hdfs/HdfsDirectory.java
@@ -20,6 +20,8 @@ package org.apache.blur.store.hdfs;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.blur.log.Log;
@@ -40,6 +42,9 @@ import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.NoLockFactory;
 
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Histogram;
+
 public class HdfsDirectory extends Directory {
 
   private static final Log LOG = LogFactory.getLog(HdfsDirectory.class);
@@ -80,10 +85,22 @@ public class HdfsDirectory extends Directory {
   private final Path path;
   private final FileSystem fileSystem;
 
+  private final Histogram readAccess;
+  private static Map<String, Histogram> readAccessMap = new ConcurrentHashMap<String,
Histogram>();
+
   public HdfsDirectory(Configuration configuration, Path path) throws IOException {
     this.path = path;
     fileSystem = path.getFileSystem(configuration);
     setLockFactory(NoLockFactory.getNoLockFactory());
+    synchronized (readAccessMap) {
+      Histogram histogram = readAccessMap.get(fileSystem.toString());
+      if (histogram == null) {
+        histogram = Metrics.newHistogram(HdfsDirectory.class, "Read Access [" + fileSystem.getUri()
+ "]");
+        readAccessMap.put(fileSystem.toString(), histogram);
+      }
+      readAccess = histogram;
+    }
+
   }
 
   @Override
@@ -96,12 +113,14 @@ public class HdfsDirectory extends Directory {
     private final long len;
     private FSDataInputStream inputStream;
     private boolean isClone;
+    private final Histogram readAccess;
 
-    public HdfsIndexInput(FileSystem fileSystem, Path filePath) throws IOException {
+    public HdfsIndexInput(FileSystem fileSystem, Path filePath, Histogram readAccess) throws
IOException {
       super(filePath.toString());
       inputStream = fileSystem.open(filePath);
       FileStatus fileStatus = fileSystem.getFileStatus(filePath);
       len = fileStatus.getLen();
+      this.readAccess = readAccess;
     }
 
     @Override
@@ -117,8 +136,11 @@ public class HdfsDirectory extends Directory {
     @Override
     protected void readInternal(byte[] b, int offset, int length) throws IOException {
       synchronized (inputStream) {
+        long start = System.nanoTime();
         inputStream.seek(getFilePointer());
         inputStream.readFully(b, offset, length);
+        long end = System.nanoTime();
+        readAccess.update((end - start) / 1000);
       }
     }
 
@@ -175,7 +197,7 @@ public class HdfsDirectory extends Directory {
       throw new FileNotFoundException("File [" + name + "] not found.");
     }
     Path filePath = getPath(name);
-    return new HdfsIndexInput(fileSystem, filePath);
+    return new HdfsIndexInput(fileSystem, filePath, readAccess);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/blur-util/pom.xml
----------------------------------------------------------------------
diff --git a/src/blur-util/pom.xml b/src/blur-util/pom.xml
index feeb135..5af96d2 100644
--- a/src/blur-util/pom.xml
+++ b/src/blur-util/pom.xml
@@ -1,22 +1,14 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied.  See the License for the
-specific language governing permissions and limitations
-under the License.
--->
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+	license agreements. See the NOTICE file distributed with this work for additional 
+	information regarding copyright ownership. The ASF licenses this file to 
+	you under the Apache License, Version 2.0 (the "License"); you may not use 
+	this file except in compliance with the License. You may obtain a copy of 
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+	by applicable law or agreed to in writing, software distributed under the 
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+	OF ANY KIND, either express or implied. See the License for the specific 
+	language governing permissions and limitations under the License. -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
@@ -24,7 +16,7 @@ under the License.
 		<groupId>org.apache.blur</groupId>
 		<artifactId>blur</artifactId>
 		<version>0.2.0-SNAPSHOT</version>
-        <relativePath>../pom.xml</relativePath>
+		<relativePath>../pom.xml</relativePath>
 	</parent>
 	<groupId>org.apache.blur</groupId>
 	<artifactId>blur-util</artifactId>
@@ -103,8 +95,14 @@ under the License.
 				</exclusion>
 			</exclusions>
 		</dependency>
+		<dependency>
+			<groupId>com.yammer.metrics</groupId>
+			<artifactId>metrics-core</artifactId>
+			<version>${metrics-core.version}</version>
+		</dependency>
 	</dependencies>
 
+
 	<repositories>
 		<repository>
 			<id>libdir</id>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0270f9a0/src/pom.xml
----------------------------------------------------------------------
diff --git a/src/pom.xml b/src/pom.xml
index 833d404..5370805 100644
--- a/src/pom.xml
+++ b/src/pom.xml
@@ -50,6 +50,7 @@ under the License.
 		<concurrentlinkedhashmap-lru.version>1.3.2</concurrentlinkedhashmap-lru.version>
 		<jline.version>2.7</jline.version>
 		<guava.version>12.0</guava.version>
+		<metrics-core.version>2.2.0</metrics-core.version>
 	</properties>
 
 	<modules>


Mime
View raw message