incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Added back in the hap usage metrics.
Date Thu, 20 Jun 2013 00:40:44 GMT
Updated Branches:
  refs/heads/master 6dd9aa3f7 -> ce716b27f


Added back in the hap usage metrics.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/ce716b27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/ce716b27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/ce716b27

Branch: refs/heads/master
Commit: ce716b27f786669eb9155cbbc6d14a2d0a97b8de
Parents: 6dd9aa3
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Jun 19 20:39:05 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Jun 19 20:39:05 2013 -0400

----------------------------------------------------------------------
 .../indexserver/DistributedIndexServer.java     | 41 +++++++++++---------
 .../java/org/apache/blur/utils/BlurUtil.java    |  6 ---
 .../java/org/apache/blur/utils/ObjectSize.java  | 37 ------------------
 .../org/apache/blur/utils/BlurUtilsTest.java    |  8 ----
 4 files changed, 22 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce716b27/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index 215d1e6..85c1810 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -51,7 +51,6 @@ import org.apache.blur.log.LogFactory;
 import org.apache.blur.lucene.search.FairSimilarity;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.lucene.store.refcounter.IndexInputCloser;
-import org.apache.blur.lucene.warmup.TraceableDirectory;
 import org.apache.blur.manager.BlurFilterCache;
 import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
@@ -81,8 +80,10 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;
@@ -94,7 +95,7 @@ import com.yammer.metrics.core.MetricName;
 public class DistributedIndexServer extends AbstractIndexServer {
 
   private static final Log LOG = LogFactory.getLog(DistributedIndexServer.class);
-  private static final long _delay = TimeUnit.SECONDS.toMillis(5);
+  private static final long _delay = TimeUnit.SECONDS.toMillis(10);
 
   private Map<String, BlurAnalyzer> _tableAnalyzers = new ConcurrentHashMap<String,
BlurAnalyzer>();
   private Map<String, TableDescriptor> _tableDescriptors = new ConcurrentHashMap<String,
TableDescriptor>();
@@ -144,7 +145,7 @@ public class DistributedIndexServer extends AbstractIndexServer {
     MetricName indexCount = new MetricName(ORG_APACHE_BLUR, BLUR, INDEX_COUNT, _cluster);
     MetricName segmentCount = new MetricName(ORG_APACHE_BLUR, BLUR, SEGMENT_COUNT, _cluster);
     MetricName indexMemoryUsage = new MetricName(ORG_APACHE_BLUR, BLUR, INDEX_MEMORY_USAGE,
_cluster);
-    
+
     Metrics.newGauge(tableCount, new AtomicLongGauge(_tableCount));
     Metrics.newGauge(indexCount, new AtomicLongGauge(_indexCount));
     Metrics.newGauge(segmentCount, new AtomicLongGauge(_segmentCount));
@@ -165,11 +166,12 @@ public class DistributedIndexServer extends AbstractIndexServer {
     _indexCloser = new BlurIndexCloser();
     _indexCloser.init();
     setupFlushCacheTimer();
-    
+
     registerMyselfAsMemberOfCluster();
     String onlineShardsPath = ZookeeperPathConstants.getOnlineShardsPath(_cluster);
     String safemodePath = ZookeeperPathConstants.getSafemodePath(_cluster);
-    SafeMode safeMode = new SafeMode(_zookeeper, safemodePath, onlineShardsPath, TimeUnit.MILLISECONDS,
_safeModeDelay, TimeUnit.SECONDS, 60);
+    SafeMode safeMode = new SafeMode(_zookeeper, safemodePath, onlineShardsPath, TimeUnit.MILLISECONDS,
_safeModeDelay,
+        TimeUnit.SECONDS, 60);
     safeMode.registerNode(getNodeName(), BlurUtil.getVersion().getBytes());
 
     _running.set(true);
@@ -250,18 +252,19 @@ public class DistributedIndexServer extends AbstractIndexServer {
       private void updateMetrics(Map<String, BlurIndex> indexes, AtomicLong segmentCount,
AtomicLong indexMemoryUsage)
           throws IOException {
         // @TODO not sure how to do this yet
-        // for (BlurIndex index : indexes.values()) {
-        // IndexReader reader = index.getIndexReader();
-        // try {
-        // IndexReader[] readers = reader.getSequentialSubReaders();
-        // if (readers != null) {
-        // segmentCount.addAndGet(readers.length);
-        // }
-        // indexMemoryUsage.addAndGet(BlurUtil.getMemoryUsage(reader));
-        // } finally {
-        // reader.decRef();
-        // }
-        // }
+
+        indexMemoryUsage.addAndGet(RamUsageEstimator.sizeOf(indexes));
+        for (BlurIndex index : indexes.values()) {
+          IndexSearcherClosable indexSearcherClosable = index.getIndexReader();
+          try {
+            IndexReader indexReader = indexSearcherClosable.getIndexReader();
+            IndexReaderContext context = indexReader.getContext();
+            segmentCount.addAndGet(context.leaves().size());
+
+          } finally {
+            indexSearcherClosable.close();
+          }
+        }
       }
     }, _delay, _delay);
   }
@@ -369,7 +372,7 @@ public class DistributedIndexServer extends AbstractIndexServer {
   @Override
   public void close() {
     if (_running.get()) {
-      
+
       _shardStateManager.close();
       _running.set(false);
       closeAllIndexes();
@@ -464,7 +467,7 @@ public class DistributedIndexServer extends AbstractIndexServer {
     } else {
       dir = directory;
     }
-    
+
     BlurIndex index;
     if (_clusterStatus.isReadOnly(true, _cluster, table)) {
       BlurIndexReader reader = new BlurIndexReader(shardContext, dir, _refresher, _indexCloser);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce716b27/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index 30efdce..ef386c2 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -89,7 +89,6 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.Code;
@@ -423,11 +422,6 @@ public class BlurUtil {
     return seconds / TimeUnit.HOURS.toSeconds(1);
   }
 
-  public static long getMemoryUsage(IndexReader r) {
-    long sizeOf = RamUsageEstimator.sizeOf(r);
-    return sizeOf;
-  }
-
   public static void createPath(ZooKeeper zookeeper, String path, byte[] data) throws KeeperException,
       InterruptedException {
     zookeeper.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce716b27/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java b/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java
deleted file mode 100644
index 2c9a2c9..0000000
--- a/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package org.apache.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.lang.instrument.Instrumentation;
-
-public class ObjectSize {
-
-  private static Instrumentation instrumentation;
-
-  public static void premain(String agentArgs, Instrumentation inst) {
-    instrumentation = inst;
-  }
-
-  public static long getSizeInBytes(Object o) {
-    if (instrumentation == null)
-      return -1;
-    if (o == null)
-      return 0;
-    return instrumentation.getObjectSize(o);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/ce716b27/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java b/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java
index 4923862..5346f6d 100644
--- a/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java
+++ b/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java
@@ -19,7 +19,6 @@ package org.apache.blur.utils;
 
 import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -92,13 +91,6 @@ public class BlurUtilsTest {
   }
 
   @Test
-  public void testMemoryUsage() throws CorruptIndexException, LockObtainFailedException,
IOException {
-    IndexReader reader = getReader();
-    long memoryUsage = BlurUtil.getMemoryUsage(reader);
-    assertTrue(memoryUsage > 0);
-  }
-
-  @Test
   public void testValidateShardCount() throws IOException {
     File file = new File(TMPDIR, "ValidateShardCount-test");
     rm(file);


Mime
View raw message