cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ble...@apache.org
Subject [6/6] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Date Tue, 28 Mar 2017 14:43:00 GMT
Merge branch 'cassandra-3.11' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/91c5f4c2
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/91c5f4c2
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/91c5f4c2

Branch: refs/heads/trunk
Commit: 91c5f4c28d87e621fee218c9a35c0d2fcb07f219
Parents: 3bd152e 689484b
Author: Benjamin Lerer <b.lerer@gmail.com>
Authored: Tue Mar 28 16:38:16 2017 +0200
Committer: Benjamin Lerer <b.lerer@gmail.com>
Committed: Tue Mar 28 16:39:40 2017 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 NEWS.txt                                        |   3 +
 .../db/commitlog/CommitLogSegment.java          |   4 +-
 .../db/commitlog/MemoryMappedSegment.java       |   4 +-
 .../cassandra/db/lifecycle/LogReplica.java      |  10 +-
 .../cassandra/hints/ChecksummedDataInput.java   |   4 +-
 .../apache/cassandra/hints/HintsCatalog.java    |   6 +-
 .../org/apache/cassandra/hints/HintsWriter.java |   6 +-
 .../cassandra/io/sstable/SSTableRewriter.java   |   4 +-
 .../io/sstable/format/SSTableReader.java        |   4 +-
 .../apache/cassandra/io/util/ChannelProxy.java  |   4 +-
 .../apache/cassandra/io/util/FileHandle.java    |   4 +-
 .../cassandra/service/CassandraDaemon.java      |   6 +-
 .../apache/cassandra/service/StartupChecks.java |  12 +-
 .../org/apache/cassandra/utils/CLibrary.java    | 403 ------------------
 .../apache/cassandra/utils/CLibraryDarwin.java  | 131 ------
 .../apache/cassandra/utils/CLibraryLinux.java   | 131 ------
 .../apache/cassandra/utils/CLibraryWindows.java | 103 -----
 .../apache/cassandra/utils/CLibraryWrapper.java |  40 --
 .../org/apache/cassandra/utils/HeapUtils.java   |   2 +-
 .../apache/cassandra/utils/NativeLibrary.java   | 407 +++++++++++++++++++
 .../cassandra/utils/NativeLibraryDarwin.java    | 129 ++++++
 .../cassandra/utils/NativeLibraryLinux.java     | 129 ++++++
 .../cassandra/utils/NativeLibraryWindows.java   | 124 ++++++
 .../cassandra/utils/NativeLibraryWrapper.java   |  44 ++
 .../org/apache/cassandra/utils/SyncUtil.java    |   6 +-
 .../org/apache/cassandra/utils/UUIDGen.java     |   2 +-
 .../apache/cassandra/utils/WindowsTimer.java    |   4 +
 .../apache/cassandra/utils/CLibraryTest.java    |  44 --
 .../cassandra/utils/NativeLibraryTest.java      |  44 ++
 30 files changed, 924 insertions(+), 891 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/91c5f4c2/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/91c5f4c2/NEWS.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/91c5f4c2/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
index 0716d47,8670c64..5edf72b
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
@@@ -37,10 -37,7 +37,10 @@@ import org.apache.cassandra.db.commitlo
  import org.apache.cassandra.db.partitions.PartitionUpdate;
  import org.apache.cassandra.io.FSWriteError;
  import org.apache.cassandra.io.util.FileUtils;
 +import org.apache.cassandra.schema.Schema;
 +import org.apache.cassandra.schema.TableId;
 +import org.apache.cassandra.schema.TableMetadata;
- import org.apache.cassandra.utils.CLibrary;
+ import org.apache.cassandra.utils.NativeLibrary;
  import org.apache.cassandra.utils.IntegerInterval;
  import org.apache.cassandra.utils.concurrent.OpOrder;
  import org.apache.cassandra.utils.concurrent.WaitQueue;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/91c5f4c2/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/91c5f4c2/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/91c5f4c2/src/java/org/apache/cassandra/service/StartupChecks.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/91c5f4c2/src/java/org/apache/cassandra/utils/NativeLibrary.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/utils/NativeLibrary.java
index 0000000,e461127..4cabf79
mode 000000,100644..100644
--- a/src/java/org/apache/cassandra/utils/NativeLibrary.java
+++ b/src/java/org/apache/cassandra/utils/NativeLibrary.java
@@@ -1,0 -1,403 +1,407 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.cassandra.utils;
+ 
+ import java.io.File;
+ import java.io.FileDescriptor;
+ import java.io.FileInputStream;
+ import java.io.IOException;
+ import java.lang.reflect.Field;
+ import java.nio.channels.FileChannel;
+ import java.util.concurrent.TimeUnit;
+ 
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ 
+ import com.sun.jna.LastErrorException;
+ import sun.nio.ch.FileChannelImpl;
+ 
+ import static org.apache.cassandra.utils.NativeLibrary.OSType.LINUX;
+ import static org.apache.cassandra.utils.NativeLibrary.OSType.MAC;
+ import static org.apache.cassandra.utils.NativeLibrary.OSType.WINDOWS;
+ import static org.apache.cassandra.utils.NativeLibrary.OSType.AIX;
+ 
+ public final class NativeLibrary
+ {
+     private static final Logger logger = LoggerFactory.getLogger(NativeLibrary.class);
+ 
+     public enum OSType
+     {
+         LINUX,
+         MAC,
+         WINDOWS,
+         AIX,
+         OTHER;
+     }
+ 
+     private static final OSType osType;
+ 
+     private static final int MCL_CURRENT;
+     private static final int MCL_FUTURE;
+ 
+     private static final int ENOMEM = 12;
+ 
+     private static final int F_GETFL   = 3;  /* get file status flags */
+     private static final int F_SETFL   = 4;  /* set file status flags */
+     private static final int F_NOCACHE = 48; /* Mac OS X specific flag, turns cache on/off
*/
+     private static final int O_DIRECT  = 040000; /* fcntl.h */
+     private static final int O_RDONLY  = 00000000; /* fcntl.h */
+ 
+     private static final int POSIX_FADV_NORMAL     = 0; /* fadvise.h */
+     private static final int POSIX_FADV_RANDOM     = 1; /* fadvise.h */
+     private static final int POSIX_FADV_SEQUENTIAL = 2; /* fadvise.h */
+     private static final int POSIX_FADV_WILLNEED   = 3; /* fadvise.h */
+     private static final int POSIX_FADV_DONTNEED   = 4; /* fadvise.h */
+     private static final int POSIX_FADV_NOREUSE    = 5; /* fadvise.h */
+ 
+     private static final NativeLibraryWrapper wrappedLibrary;
+     private static boolean jnaLockable = false;
+ 
+     private static final Field FILE_DESCRIPTOR_FD_FIELD;
+     private static final Field FILE_CHANNEL_FD_FIELD;
+ 
+     static
+     {
+         FILE_DESCRIPTOR_FD_FIELD = FBUtilities.getProtectedField(FileDescriptor.class, "fd");
+         FILE_CHANNEL_FD_FIELD = FBUtilities.getProtectedField(FileChannelImpl.class, "fd");
+ 
+         // detect the OS type the JVM is running on and then set the CLibraryWrapper
+         // instance to a compatable implementation of CLibraryWrapper for that OS type
+         osType = getOsType();
+         switch (osType)
+         {
+             case MAC: wrappedLibrary = new NativeLibraryDarwin(); break;
+             case WINDOWS: wrappedLibrary = new NativeLibraryWindows(); break;
+             case LINUX:
+             case AIX:
+             case OTHER:
+             default: wrappedLibrary = new NativeLibraryLinux();
+         }
+ 
+         if (System.getProperty("os.arch").toLowerCase().contains("ppc"))
+         {
+             if (osType == LINUX)
+             {
+                MCL_CURRENT = 0x2000;
+                MCL_FUTURE = 0x4000;
+             }
+             else if (osType == AIX)
+             {
+                 MCL_CURRENT = 0x100;
+                 MCL_FUTURE = 0x200;
+             }
+             else
+             {
+                 MCL_CURRENT = 1;
+                 MCL_FUTURE = 2;
+             }
+         }
+         else
+         {
+             MCL_CURRENT = 1;
+             MCL_FUTURE = 2;
+         }
+     }
+ 
+     private NativeLibrary() {}
+ 
+     /**
+      * @return the detected OSType of the Operating System running the JVM using crude string
matching
+      */
+     private static OSType getOsType()
+     {
+         String osName = System.getProperty("os.name").toLowerCase();
 -        if (osName.contains("mac"))
++        if  (osName.contains("linux"))
++            return LINUX;
++        else if (osName.contains("mac"))
+             return MAC;
+         else if (osName.contains("windows"))
+             return WINDOWS;
 -        else if (osName.contains("aix"))
++
++        logger.warn("the current operating system, {}, is unsupported by cassandra", osName);
++        if (osName.contains("aix"))
+             return AIX;
+         else
+             // fall back to the Linux impl for all unknown OS types until otherwise implicitly
supported as needed
+             return LINUX;
+     }
+ 
+     private static int errno(RuntimeException e)
+     {
+         assert e instanceof LastErrorException;
+         try
+         {
+             return ((LastErrorException) e).getErrorCode();
+         }
+         catch (NoSuchMethodError x)
+         {
+             logger.warn("Obsolete version of JNA present; unable to read errno. Upgrade
to JNA 3.2.7 or later");
+             return 0;
+         }
+     }
+ 
+     /**
+      * Checks if the library has been successfully linked.
+      * @return {@code true} if the library has been successfully linked, {@code false} otherwise.
+      */
+     public static boolean isAvailable()
+     {
+         return wrappedLibrary.isAvailable();
+     }
+ 
+     public static boolean jnaMemoryLockable()
+     {
+         return jnaLockable;
+     }
+ 
+     public static void tryMlockall()
+     {
+         try
+         {
+             wrappedLibrary.callMlockall(MCL_CURRENT);
+             jnaLockable = true;
+             logger.info("JNA mlockall successful");
+         }
+         catch (UnsatisfiedLinkError e)
+         {
+             // this will have already been logged by CLibrary, no need to repeat it
+         }
+         catch (RuntimeException e)
+         {
+             if (!(e instanceof LastErrorException))
+                 throw e;
+ 
+             if (errno(e) == ENOMEM && osType == LINUX)
+             {
+                 logger.warn("Unable to lock JVM memory (ENOMEM)."
+                         + " This can result in part of the JVM being swapped out, especially
with mmapped I/O enabled."
+                         + " Increase RLIMIT_MEMLOCK or run Cassandra as root.");
+             }
+             else if (osType != MAC)
+             {
+                 // OS X allows mlockall to be called, but always returns an error
+                 logger.warn("Unknown mlockall error {}", errno(e));
+             }
+         }
+     }
+ 
+     public static void trySkipCache(String path, long offset, long len)
+     {
+         File f = new File(path);
+         if (!f.exists())
+             return;
+ 
+         try (FileInputStream fis = new FileInputStream(f))
+         {
+             trySkipCache(getfd(fis.getChannel()), offset, len, path);
+         }
+         catch (IOException e)
+         {
+             logger.warn("Could not skip cache", e);
+         }
+     }
+ 
+     public static void trySkipCache(int fd, long offset, long len, String path)
+     {
+         if (len == 0)
+             trySkipCache(fd, 0, 0, path);
+ 
+         while (len > 0)
+         {
+             int sublen = (int) Math.min(Integer.MAX_VALUE, len);
+             trySkipCache(fd, offset, sublen, path);
+             len -= sublen;
+             offset -= sublen;
+         }
+     }
+ 
+     public static void trySkipCache(int fd, long offset, int len, String path)
+     {
+         if (fd < 0)
+             return;
+ 
+         try
+         {
+             if (osType == LINUX)
+             {
+                 int result = wrappedLibrary.callPosixFadvise(fd, offset, len, POSIX_FADV_DONTNEED);
+                 if (result != 0)
+                     NoSpamLogger.log(
+                             logger,
+                             NoSpamLogger.Level.WARN,
+                             10,
+                             TimeUnit.MINUTES,
+                             "Failed trySkipCache on file: {} Error: " + wrappedLibrary.callStrerror(result).getString(0),
+                             path);
+             }
+         }
+         catch (UnsatisfiedLinkError e)
+         {
+             // if JNA is unavailable just skipping Direct I/O
+             // instance of this class will act like normal RandomAccessFile
+         }
+         catch (RuntimeException e)
+         {
+             if (!(e instanceof LastErrorException))
+                 throw e;
+ 
+             logger.warn("posix_fadvise({}, {}) failed, errno ({}).", fd, offset, errno(e));
+         }
+     }
+ 
+     public static int tryFcntl(int fd, int command, int flags)
+     {
+         // fcntl return value may or may not be useful, depending on the command
+         int result = -1;
+ 
+         try
+         {
+             result = wrappedLibrary.callFcntl(fd, command, flags);
+         }
+         catch (UnsatisfiedLinkError e)
+         {
+             // if JNA is unavailable just skipping
+         }
+         catch (RuntimeException e)
+         {
+             if (!(e instanceof LastErrorException))
+                 throw e;
+ 
+             logger.warn("fcntl({}, {}, {}) failed, errno ({}).", fd, command, flags, errno(e));
+         }
+ 
+         return result;
+     }
+ 
+     public static int tryOpenDirectory(String path)
+     {
+         int fd = -1;
+ 
+         try
+         {
+             return wrappedLibrary.callOpen(path, O_RDONLY);
+         }
+         catch (UnsatisfiedLinkError e)
+         {
+             // JNA is unavailable just skipping Direct I/O
+         }
+         catch (RuntimeException e)
+         {
+             if (!(e instanceof LastErrorException))
+                 throw e;
+ 
+             logger.warn("open({}, O_RDONLY) failed, errno ({}).", path, errno(e));
+         }
+ 
+         return fd;
+     }
+ 
+     public static void trySync(int fd)
+     {
+         if (fd == -1)
+             return;
+ 
+         try
+         {
+             wrappedLibrary.callFsync(fd);
+         }
+         catch (UnsatisfiedLinkError e)
+         {
+             // JNA is unavailable just skipping Direct I/O
+         }
+         catch (RuntimeException e)
+         {
+             if (!(e instanceof LastErrorException))
+                 throw e;
+ 
+             logger.warn("fsync({}) failed, errorno ({}) {}", fd, errno(e), e);
+         }
+     }
+ 
+     public static void tryCloseFD(int fd)
+     {
+         if (fd == -1)
+             return;
+ 
+         try
+         {
+             wrappedLibrary.callClose(fd);
+         }
+         catch (UnsatisfiedLinkError e)
+         {
+             // JNA is unavailable just skipping Direct I/O
+         }
+         catch (RuntimeException e)
+         {
+             if (!(e instanceof LastErrorException))
+                 throw e;
+ 
+             logger.warn("close({}) failed, errno ({}).", fd, errno(e));
+         }
+     }
+ 
+     public static int getfd(FileChannel channel)
+     {
+         try
+         {
+             return getfd((FileDescriptor)FILE_CHANNEL_FD_FIELD.get(channel));
+         }
+         catch (IllegalArgumentException|IllegalAccessException e)
+         {
+             logger.warn("Unable to read fd field from FileChannel");
+         }
+         return -1;
+     }
+ 
+     /**
+      * Get system file descriptor from FileDescriptor object.
+      * @param descriptor - FileDescriptor objec to get fd from
+      * @return file descriptor, -1 or error
+      */
+     public static int getfd(FileDescriptor descriptor)
+     {
+         try
+         {
+             return FILE_DESCRIPTOR_FD_FIELD.getInt(descriptor);
+         }
+         catch (Exception e)
+         {
+             JVMStabilityInspector.inspectThrowable(e);
+             logger.warn("Unable to read fd field from FileDescriptor");
+         }
+ 
+         return -1;
+     }
+ 
+     /**
+      * @return the PID of the JVM or -1 if we failed to get the PID
+      */
+     public static long getProcessID()
+     {
+         try
+         {
+             return wrappedLibrary.callGetpid();
+         }
+         catch (Exception e)
+         {
+             logger.info("Failed to get PID from JNA", e);
+         }
+ 
+         return -1;
+     }
+ }


Mime
View raw message