hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mbau...@apache.org
Subject svn commit: r1377223 - in /hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util: FSRegionScanner.java FSUtils.java
Date Sat, 25 Aug 2012 06:47:41 GMT
Author: mbautin
Date: Sat Aug 25 06:47:41 2012
New Revision: 1377223

URL: http://svn.apache.org/viewvc?rev=1377223&view=rev
Log:
[jira] [HBASE-6519] [89-fb] FSRegionScanner should be in its own file

Author: rvadali

Summary:
I see class not found errors when starting the master on my laptop.
It also seems better to not have more than one top-level class in the
same file.

Test Plan:
Single-node setup.

Revert Plan:

Tags:

- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -

Reviewers: kannan, kranganathan

CC: hbase-eng@lists

Differential Revision: https://phabricator.fb.com/D540344

Added:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java?rev=1377223&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSRegionScanner.java
Sat Aug 25 06:47:41 2012
@@ -0,0 +1,168 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Thread to be used for
+ */
+class FSRegionScanner implements Runnable {
+  static private final Log LOG = LogFactory.getLog(FSRegionScanner.class);
+
+  private Path regionPath;
+
+  /**
+   * The file system used
+   */
+  private FileSystem fs;
+
+  /**
+   * Maps each region to the RS with highest locality for that region.
+   */
+  private MapWritable regionToBestLocalityRSMapping;
+
+  /**
+   * Maps region encoded names to maps of hostnames to fractional locality of
+   * that region on that host.
+   */
+  private Map<String, Map<String, Float>> regionDegreeLocalityMapping;
+
+  FSRegionScanner(FileSystem fs, Path regionPath,
+                  MapWritable regionToBestLocalityRSMapping,
+                  Map<String, Map<String, Float>> regionDegreeLocalityMapping)
{
+    this.fs = fs;
+    this.regionPath = regionPath;
+    this.regionToBestLocalityRSMapping = regionToBestLocalityRSMapping;
+    this.regionDegreeLocalityMapping = regionDegreeLocalityMapping;
+  }
+
+  @Override
+  public void run() {
+    try {
+      // empty the map for each region
+      Map<String, AtomicInteger> blockCountMap = new HashMap<String, AtomicInteger>();
+
+      //get table name
+      String tableName = regionPath.getParent().getName();
+      int totalBlkCount = 0;
+
+      // ignore null
+      FileStatus[] cfList = fs.listStatus(regionPath);
+      if (null == cfList) {
+        return;
+      }
+
+      // for each cf, get all the blocks information
+      for (FileStatus cfStatus : cfList) {
+        if (!cfStatus.isDir()) {
+          // skip because this is not a CF directory
+          continue;
+        }
+        if (cfStatus.getPath().getName().startsWith(".")) {
+          continue;
+        }
+        FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath());
+        if (null == storeFileLists) {
+          continue;
+        }
+
+        for (FileStatus storeFile : storeFileLists) {
+          BlockLocation[] blkLocations =
+            fs.getFileBlockLocations(storeFile, 0, storeFile.getLen());
+          if (null == blkLocations) {
+            continue;
+          }
+
+          totalBlkCount += blkLocations.length;
+          for(BlockLocation blk: blkLocations) {
+            for (String host: blk.getHosts()) {
+              AtomicInteger count = blockCountMap.get(host);
+              if (count == null) {
+                count = new AtomicInteger(0);
+                blockCountMap.put(host, count);
+              }
+              count.incrementAndGet();
+            }
+          }
+        }
+      }
+
+      if (regionToBestLocalityRSMapping != null) {
+        int largestBlkCount = 0;
+        String hostToRun = null;
+        for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
+          String host = entry.getKey();
+
+          int tmp = entry.getValue().get();
+          if (tmp > largestBlkCount) {
+            largestBlkCount = tmp;
+            hostToRun = host;
+          }
+        }
+
+        // empty regions could make this null
+        if (null == hostToRun) {
+          return;
+        }
+
+        if (hostToRun.endsWith(".")) {
+          hostToRun = hostToRun.substring(0, hostToRun.length()-1);
+        }
+        String name = tableName + ":" + regionPath.getName();
+        synchronized (regionToBestLocalityRSMapping) {
+          regionToBestLocalityRSMapping.put(new Text(name), new Text(hostToRun));
+        }
+      }
+
+      if (regionDegreeLocalityMapping != null && totalBlkCount > 0) {
+        Map<String, Float> hostLocalityMap = new HashMap<String, Float>();
+        for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
+          String host = entry.getKey();
+          if (host.endsWith(".")) {
+            host = host.substring(0, host.length() - 1);
+          }
+          // Locality is fraction of blocks local to this host.
+          float locality = ((float)entry.getValue().get()) / totalBlkCount;
+          hostLocalityMap.put(host, locality);
+        }
+        // Put the locality map into the result map, keyed by the encoded name
+        // of the region.
+        regionDegreeLocalityMapping.put(regionPath.getName(), hostLocalityMap);
+      }
+    } catch (IOException e) {
+      LOG.warn("Problem scanning file system", e);
+    } catch (RuntimeException e) {
+      LOG.warn("Problem scanning file system", e);
+    }
+  }
+}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1377223&r1=1377222&r2=1377223&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Sat Aug
25 06:47:41 2012
@@ -32,12 +32,10 @@ import java.util.concurrent.ArrayBlockin
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -55,7 +53,6 @@ import org.apache.hadoop.hdfs.protocol.F
 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
 
 /**
  * Utility methods for interacting with the underlying file system.
@@ -1018,136 +1015,3 @@ public class FSUtils {
 
 }
 
-/**
- * Thread to be used for
- */
-class FSRegionScanner implements Runnable {
-  static private final Log LOG = LogFactory.getLog(FSRegionScanner.class);
-
-  private Path regionPath;
-
-  /**
-   * The file system used
-   */
-  private FileSystem fs;
-
-  /**
-   * Maps each region to the RS with highest locality for that region.
-   */
-  private MapWritable regionToBestLocalityRSMapping;
-
-  /**
-   * Maps region encoded names to maps of hostnames to fractional locality of
-   * that region on that host.
-   */
-  private Map<String, Map<String, Float>> regionDegreeLocalityMapping;
-
-  FSRegionScanner(FileSystem fs, Path regionPath,
-      MapWritable regionToBestLocalityRSMapping,
-      Map<String, Map<String, Float>> regionDegreeLocalityMapping) {
-    this.fs = fs;
-    this.regionPath = regionPath;
-    this.regionToBestLocalityRSMapping = regionToBestLocalityRSMapping;
-    this.regionDegreeLocalityMapping = regionDegreeLocalityMapping;
-  }
-
-  @Override
-  public void run() {
-    try {
-      // empty the map for each region
-      Map<String, AtomicInteger> blockCountMap = new HashMap<String, AtomicInteger>();
-
-      //get table name
-      String tableName = regionPath.getParent().getName();
-      int totalBlkCount = 0;
-
-      // ignore null
-      FileStatus[] cfList = fs.listStatus(regionPath);
-      if (null == cfList) {
-        return;
-      }
-
-      // for each cf, get all the blocks information
-      for (FileStatus cfStatus : cfList) {
-        if (!cfStatus.isDir()) {
-          // skip because this is not a CF directory
-          continue;
-        }
-        if (cfStatus.getPath().getName().startsWith(".")) {
-          continue;
-        }
-        FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath());
-        if (null == storeFileLists) {
-          continue;
-        }
-
-        for (FileStatus storeFile : storeFileLists) {
-          BlockLocation[] blkLocations =
-            fs.getFileBlockLocations(storeFile, 0, storeFile.getLen());
-          if (null == blkLocations) {
-            continue;
-          }
-
-          totalBlkCount += blkLocations.length;
-          for(BlockLocation blk: blkLocations) {
-            for (String host: blk.getHosts()) {
-              AtomicInteger count = blockCountMap.get(host);
-              if (count == null) {
-                count = new AtomicInteger(0);
-                blockCountMap.put(host, count);
-              }
-             count.incrementAndGet();
-            }
-          }
-        }
-      }
-
-      if (regionToBestLocalityRSMapping != null) {
-        int largestBlkCount = 0;
-        String hostToRun = null;
-        for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
-          String host = entry.getKey();
-
-          int tmp = entry.getValue().get();
-          if (tmp > largestBlkCount) {
-            largestBlkCount = tmp;
-            hostToRun = host;
-          }
-        }
-
-        // empty regions could make this null
-        if (null == hostToRun) {
-          return;
-        }
-
-        if (hostToRun.endsWith(".")) {
-          hostToRun = hostToRun.substring(0, hostToRun.length()-1);
-        }
-        String name = tableName + ":" + regionPath.getName();
-        synchronized (regionToBestLocalityRSMapping) {
-          regionToBestLocalityRSMapping.put(new Text(name), new Text(hostToRun));
-        }
-      }
-
-      if (regionDegreeLocalityMapping != null && totalBlkCount > 0) {
-        Map<String, Float> hostLocalityMap = new HashMap<String, Float>();
-        for (Map.Entry<String, AtomicInteger> entry : blockCountMap.entrySet()) {
-          String host = entry.getKey();
-          if (host.endsWith(".")) {
-            host = host.substring(0, host.length() - 1);
-          }
-          // Locality is fraction of blocks local to this host.
-          float locality = ((float)entry.getValue().get()) / totalBlkCount;
-          hostLocalityMap.put(host, locality);
-        }
-        // Put the locality map into the result map, keyed by the encoded name
-        // of the region.
-        regionDegreeLocalityMapping.put(regionPath.getName(), hostLocalityMap);
-      }
-    } catch (IOException e) {
-      LOG.warn("Problem scanning file system", e);
-    } catch (RuntimeException e) {
-      LOG.warn("Problem scanning file system", e);
-    }
-  }
-}



Mime
View raw message