accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [1/8] accumulo git commit: ACCUMULO-4080 Only check /dev/ entries and FSes in [ext3, ext4, xfs].
Date Fri, 18 Dec 2015 19:08:09 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.6 4f02a65d5 -> d3d825b15
  refs/heads/1.7 f51e6b0c2 -> 5960cfe99
  refs/heads/master 794ec90d5 -> 25af3a3a5


ACCUMULO-4080 Only check /dev/ entries and FSes in [ext3, ext4, xfs].

By too aggressively monitoring filesystem mounts, we can actually
make accumulo unstable. Try to restrict what we monitor.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d3d825b1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d3d825b1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d3d825b1

Branch: refs/heads/1.6
Commit: d3d825b1517f9eb955c2285d1dd39ebfea2c7595
Parents: 4f02a65
Author: Josh Elser <elserj@apache.org>
Authored: Thu Dec 17 15:12:41 2015 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Thu Dec 17 15:12:41 2015 -0500

----------------------------------------------------------------------
 .../accumulo/server/util/FileSystemMonitor.java | 58 +++++++-----
 .../server/util/FileSystemMonitorTest.java      | 98 ++++++++++++++++++++
 2 files changed, 134 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/d3d825b1/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
b/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
index 94a80d9..e2e1d46 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileSystemMonitor.java
@@ -32,41 +32,66 @@ import java.util.TimerTask;
 
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
 
 public class FileSystemMonitor {
   private static final String PROC_MOUNTS = "/proc/mounts";
-  private static final Logger log = Logger.getLogger(FileSystemMonitor.class);
+  private static final Logger log = LoggerFactory.getLogger(FileSystemMonitor.class);
+  private static final String DEVICE_PREFIX = "/dev/";
+  private static final Set<String> ACCEPTED_FILESYSTEMS = Sets.newHashSet("ext3", "ext4",
"xfs");
 
-  private static class Mount {
+  static class Mount {
+    String device;
     String mountPoint;
+    String filesystemType;
     Set<String> options;
 
     Mount(String line) {
       String tokens[] = line.split("\\s+");
 
-      mountPoint = tokens[1];
+      device = tokens[0].trim();
+      mountPoint = tokens[1].trim();
+      filesystemType = tokens[2].trim().toLowerCase();
 
       options = new HashSet<String>(Arrays.asList(tokens[3].split(",")));
     }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder(32);
+      sb.append(device).append(" on ").append(mountPoint).append(" as ").append(filesystemType)
+          .append(" with options ").append(options);
+      return sb.toString();
+    }
   }
 
   static List<Mount> parse(String procFile) throws IOException {
 
-    List<Mount> mounts = new ArrayList<Mount>();
-
     FileReader fr = new FileReader(procFile);
     BufferedReader br = new BufferedReader(fr);
 
-    String line;
     try {
-      while ((line = br.readLine()) != null)
-        mounts.add(new Mount(line));
+      return getMountsFromFile(br);
     } finally {
       br.close();
     }
+  }
 
+  static List<Mount> getMountsFromFile(BufferedReader br) throws IOException {
+    List<Mount> mounts = new ArrayList<Mount>();
+    String line;
+    while ((line = br.readLine()) != null) {
+      Mount mount = new Mount(line);
+      if (mount.device.startsWith(DEVICE_PREFIX) && ACCEPTED_FILESYSTEMS.contains(mount.filesystemType))
{
+        log.trace("Retaining mount to check: '{}'", mount);
+        mounts.add(mount);
+      } else {
+        log.trace("Ignoring mount to check: '{}'", mount);
+      }
+    }
     return mounts;
   }
 
@@ -92,7 +117,7 @@ public class FileSystemMonitor {
         } catch (final Exception e) {
           Halt.halt(-42, new Runnable() {
             public void run() {
-              log.fatal("Exception while checking mount points, halting process", e);
+              log.error("Exception while checking mount points, halting process", e);
             }
           });
         }
@@ -108,17 +133,6 @@ public class FileSystemMonitor {
 
   }
 
-  protected void logAsync(final Level level, final String msg, final Exception e) {
-    Runnable r = new Runnable() {
-      @Override
-      public void run() {
-        log.log(level, msg, e);
-      }
-    };
-
-    new Thread(r).start();
-  }
-
   protected void checkMounts(String procFile) throws Exception {
     List<Mount> mounts = parse(procFile);
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/d3d825b1/server/base/src/test/java/org/apache/accumulo/server/util/FileSystemMonitorTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/FileSystemMonitorTest.java
b/server/base/src/test/java/org/apache/accumulo/server/util/FileSystemMonitorTest.java
new file mode 100644
index 0000000..7d4bd20
--- /dev/null
+++ b/server/base/src/test/java/org/apache/accumulo/server/util/FileSystemMonitorTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.server.util.FileSystemMonitor.Mount;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileSystemMonitorTest {
+  private static final Logger log = LoggerFactory.getLogger(FileSystemMonitorTest.class);
+  private static final Charset UTF_8 = Charset.forName("UTF-8");
+  
+  @Test
+  public void testFilteredMountEntries() throws Exception {
+    String[] mountEntries = new String[] {
+        "rootfs / rootfs rw 0 0",
+        "proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0",
+        "sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0",
+        "devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=8119336k,nr_inodes=2029834,mode=755
0 0",
+        "securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0",
+        "tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0",
+        "devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000
0 0",
+        "tmpfs /run tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0",
+        "tmpfs /sys/fs/cgroup tmpfs ro,seclabel,nosuid,nodev,noexec,mode=755 0 0",
+        "cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd
0 0",
+        "pstore /sys/fs/pstore pstore rw,nosuid,nodev,noexec,relatime 0 0",
+        "cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0",
+        "cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpuacct,cpu
0 0",
+        "cgroup /sys/fs/cgroup/memory cgroup rw,nosuid,nodev,noexec,relatime,memory 0 0",
+        "cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0",
+        "cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0",
+        "cgroup /sys/fs/cgroup/net_cls cgroup rw,nosuid,nodev,noexec,relatime,net_cls 0 0",
+        "cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0",
+        "cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event
0 0",
+        "cgroup /sys/fs/cgroup/hugetlb cgroup rw,nosuid,nodev,noexec,relatime,hugetlb 0 0",
+        "configfs /sys/kernel/config configfs rw,relatime 0 0",
+        "/dev/vda1 / xfs rw,seclabel,relatime,attr2,inode64,noquota 0 0",
+        "/dev/vda2 /ignoreme reiserfs rw 0 0",
+        "rpc_pipefs /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0",
+        "selinuxfs /sys/fs/selinux selinuxfs rw,relatime 0 0",
+        "systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=32,pgrp=1,timeout=300,minproto=5,maxproto=5,direct
0 0",
+        "debugfs /sys/kernel/debug debugfs rw,relatime 0 0",
+        "mqueue /dev/mqueue mqueue rw,seclabel,relatime 0 0",
+        "hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime 0 0",
+        "sunrpc /proc/fs/nfsd nfsd rw,relatime 0 0",
+        "/dev/vdb /grid/0 ext4 rw,seclabel,relatime,data=ordered 0 0"
+    };
+
+    StringBuilder sb = new StringBuilder(256);
+    for (String mountEntry : mountEntries) {
+      if (sb.length() > 0) {
+        sb.append("\n");
+      }
+      sb.append(mountEntry);
+    }
+
+    ByteArrayInputStream is = new ByteArrayInputStream(sb.toString().getBytes(UTF_8));
+    BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+
+    List<Mount> mounts = FileSystemMonitor.getMountsFromFile(reader);
+    log.info("Filtered mount points: " + mounts);
+    assertEquals(2, mounts.size());
+    Set<String> expectedCheckedMountPoints = new HashSet<String>();
+    expectedCheckedMountPoints.add("/");
+    expectedCheckedMountPoints.add("/grid/0");
+    for (Mount mount : mounts) {
+      assertTrue("Did not expect to find " + mount, expectedCheckedMountPoints.remove(mount.mountPoint));
+    }
+    assertEquals("Should not have any extra mount points: " + expectedCheckedMountPoints,
0, expectedCheckedMountPoints.size());
+  }
+  
+}


Mime
View raw message