cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [2/3] git commit: Fix direct Memory on architectures that do not support unaligned long access patch by Dmitry Shohov and Benedict Elliott Smith for CASSANDRA-6628
Date Tue, 04 Feb 2014 05:51:28 GMT
Fix direct Memory on architectures that do not support unaligned long access
patch by Dmitry Shohov and Benedict Elliott Smith for CASSANDRA-6628


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/039e9b9a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/039e9b9a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/039e9b9a

Branch: refs/heads/trunk
Commit: 039e9b9a18cbe78091231a4538b6d428deacc771
Parents: 066d00b
Author: Jonathan Ellis <jbellis@apache.org>
Authored: Mon Feb 3 23:50:08 2014 -0600
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Mon Feb 3 23:50:08 2014 -0600

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 .../cassandra/config/DatabaseDescriptor.java    |  20 ++-
 .../org/apache/cassandra/io/util/Memory.java    | 123 ++++++++++++++++++-
 .../cassandra/service/CassandraDaemon.java      |   2 +-
 .../cassandra/utils/FastByteComparisons.java    |   6 +
 5 files changed, 145 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/039e9b9a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4440942..b1fade1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,6 @@
 2.0.6
+ * Fix direct Memory on architectures that do not support unaligned long access
+   (CASSANDRA-6628)
  * Let scrub optionally skip broken counter partitions (CASSANDRA-5930)
 
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/039e9b9a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 44d9d3a..bd5db69 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -172,12 +172,12 @@ public class DatabaseDescriptor
         }
 
         if (conf.commitlog_total_space_in_mb == null)
-            conf.commitlog_total_space_in_mb = System.getProperty("os.arch").contains("64")
? 1024 : 32;
+            conf.commitlog_total_space_in_mb = hasLargeAddressSpace() ? 1024 : 32;
 
         /* evaluate the DiskAccessMode Config directive, which also affects indexAccessMode
selection */
         if (conf.disk_access_mode == Config.DiskAccessMode.auto)
         {
-            conf.disk_access_mode = System.getProperty("os.arch").contains("64") ? Config.DiskAccessMode.mmap
: Config.DiskAccessMode.standard;
+            conf.disk_access_mode = hasLargeAddressSpace() ? Config.DiskAccessMode.mmap :
Config.DiskAccessMode.standard;
             indexAccessMode = conf.disk_access_mode;
             logger.info("DiskAccessMode 'auto' determined to be " + conf.disk_access_mode
+ ", indexAccessMode is " + indexAccessMode );
         }
@@ -1323,4 +1323,20 @@ public class DatabaseDescriptor
             throw new RuntimeException(e);
         }
     }
+
+    public static boolean hasLargeAddressSpace()
+    {
+        // currently we just check if it's a 64bit arch, but any we only really care if the
address space is large
+        String datamodel = System.getProperty("sun.arch.data.model");
+        if (datamodel != null)
+        {
+            switch (datamodel)
+            {
+                case "64": return true;
+                case "32": return false;
+            }
+        }
+        String arch = System.getProperty("os.arch");
+        return arch.contains("64") || arch.contains("sparcv9");
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/039e9b9a/src/java/org/apache/cassandra/io/util/Memory.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/Memory.java b/src/java/org/apache/cassandra/io/util/Memory.java
index f276190..263205b 100644
--- a/src/java/org/apache/cassandra/io/util/Memory.java
+++ b/src/java/org/apache/cassandra/io/util/Memory.java
@@ -17,9 +17,10 @@
  */
 package org.apache.cassandra.io.util;
 
-import sun.misc.Unsafe;
+import java.nio.ByteOrder;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import sun.misc.Unsafe;
 
 /**
  * An off-heap region of memory that must be manually free'd when no longer needed.
@@ -30,6 +31,16 @@ public class Memory
     private static final IAllocator allocator = DatabaseDescriptor.getoffHeapMemoryAllocator();
     private static final long BYTE_ARRAY_BASE_OFFSET = unsafe.arrayBaseOffset(byte[].class);
 
+    private static final boolean bigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN);
+    private static final boolean unaligned;
+
+    static
+    {
+        String arch = System.getProperty("os.arch");
+        unaligned = arch.equals("i386") || arch.equals("x86")
+                    || arch.equals("amd64") || arch.equals("x86_64");
+    }
+
     protected long peer;
     // size of the memory region
     private final long size;
@@ -64,13 +75,71 @@ public class Memory
     public void setLong(long offset, long l)
     {
         checkPosition(offset);
-        unsafe.putLong(peer + offset, l);
+        if (unaligned)
+        {
+            unsafe.putLong(peer + offset, l);
+        }
+        else
+        {
+            putLongByByte(peer + offset, l);
+        }
+    }
+
+    private void putLongByByte(long address, long value)
+    {
+        if (bigEndian)
+        {
+            unsafe.putByte(address, (byte) (value >> 56));
+            unsafe.putByte(address + 1, (byte) (value >> 48));
+            unsafe.putByte(address + 2, (byte) (value >> 40));
+            unsafe.putByte(address + 3, (byte) (value >> 32));
+            unsafe.putByte(address + 4, (byte) (value >> 24));
+            unsafe.putByte(address + 5, (byte) (value >> 16));
+            unsafe.putByte(address + 6, (byte) (value >> 8));
+            unsafe.putByte(address + 7, (byte) (value));
+        }
+        else
+        {
+            unsafe.putByte(address + 7, (byte) (value >> 56));
+            unsafe.putByte(address + 6, (byte) (value >> 48));
+            unsafe.putByte(address + 5, (byte) (value >> 40));
+            unsafe.putByte(address + 4, (byte) (value >> 32));
+            unsafe.putByte(address + 3, (byte) (value >> 24));
+            unsafe.putByte(address + 2, (byte) (value >> 16));
+            unsafe.putByte(address + 1, (byte) (value >> 8));
+            unsafe.putByte(address, (byte) (value));
+        }
     }
 
     public void setInt(long offset, int l)
     {
         checkPosition(offset);
-        unsafe.putInt(peer + offset, l);
+        if (unaligned)
+        {
+            unsafe.putInt(peer + offset, l);
+        }
+        else
+        {
+            putIntByByte(peer + offset, l);
+        }
+    }
+
+    private void putIntByByte(long address, int value)
+    {
+        if (bigEndian)
+        {
+            unsafe.putByte(address, (byte) (value >> 24));
+            unsafe.putByte(address + 1, (byte) (value >> 16));
+            unsafe.putByte(address + 2, (byte) (value >> 8));
+            unsafe.putByte(address + 3, (byte) (value));
+        }
+        else
+        {
+            unsafe.putByte(address + 3, (byte) (value >> 24));
+            unsafe.putByte(address + 2, (byte) (value >> 16));
+            unsafe.putByte(address + 1, (byte) (value >> 8));
+            unsafe.putByte(address, (byte) (value));
+        }
     }
 
     /**
@@ -108,13 +177,57 @@ public class Memory
     public long getLong(long offset)
     {
         checkPosition(offset);
-        return unsafe.getLong(peer + offset);
+        if (unaligned) {
+            return unsafe.getLong(peer + offset);
+        } else {
+            return getLongByByte(peer + offset);
+        }
+    }
+
+    private long getLongByByte(long address) {
+        if (bigEndian) {
+            return  (((long) unsafe.getByte(address    )       ) << 56) |
+                    (((long) unsafe.getByte(address + 1) & 0xff) << 48) |
+                    (((long) unsafe.getByte(address + 2) & 0xff) << 40) |
+                    (((long) unsafe.getByte(address + 3) & 0xff) << 32) |
+                    (((long) unsafe.getByte(address + 4) & 0xff) << 24) |
+                    (((long) unsafe.getByte(address + 5) & 0xff) << 16) |
+                    (((long) unsafe.getByte(address + 6) & 0xff) <<  8) |
+                    (((long) unsafe.getByte(address + 7) & 0xff)      );
+        } else {
+            return  (((long) unsafe.getByte(address + 7)       ) << 56) |
+                    (((long) unsafe.getByte(address + 6) & 0xff) << 48) |
+                    (((long) unsafe.getByte(address + 5) & 0xff) << 40) |
+                    (((long) unsafe.getByte(address + 4) & 0xff) << 32) |
+                    (((long) unsafe.getByte(address + 3) & 0xff) << 24) |
+                    (((long) unsafe.getByte(address + 2) & 0xff) << 16) |
+                    (((long) unsafe.getByte(address + 1) & 0xff) <<  8) |
+                    (((long) unsafe.getByte(address    ) & 0xff)      );
+        }
     }
 
     public int getInt(long offset)
     {
         checkPosition(offset);
-        return unsafe.getInt(peer + offset);
+        if (unaligned) {
+            return unsafe.getInt(peer + offset);
+        } else {
+            return getIntByByte(peer + offset);
+        }
+    }
+
+    private int getIntByByte(long address) {
+        if (bigEndian) {
+            return  (((int) unsafe.getByte(address    )       ) << 24) |
+                    (((int) unsafe.getByte(address + 1) & 0xff) << 16) |
+                    (((int) unsafe.getByte(address + 2) & 0xff) << 8 ) |
+                    (((int) unsafe.getByte(address + 3) & 0xff)      );
+        } else {
+            return  (((int) unsafe.getByte(address + 3)       ) << 24) |
+                    (((int) unsafe.getByte(address + 2) & 0xff) << 16) |
+                    (((int) unsafe.getByte(address + 1) & 0xff) <<  8) |
+                    (((int) unsafe.getByte(address    ) & 0xff)      );
+        }
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/039e9b9a/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 60e9c25..9e63c05 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -148,7 +148,7 @@ public class CassandraDaemon
     protected void setup()
     {
         // log warnings for different kinds of sub-optimal JVMs.  tldr use 64-bit Oracle
>= 1.6u32
-        if (!System.getProperty("os.arch").contains("64"))
+        if (!DatabaseDescriptor.hasLargeAddressSpace())
             logger.info("32bit JVM detected.  It is recommended to run Cassandra on a 64bit
JVM for better performance.");
         String javaVersion = System.getProperty("java.version");
         String javaVmName = System.getProperty("java.vm.name");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/039e9b9a/src/java/org/apache/cassandra/utils/FastByteComparisons.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/FastByteComparisons.java b/src/java/org/apache/cassandra/utils/FastByteComparisons.java
index c135a01..4be6cd4 100644
--- a/src/java/org/apache/cassandra/utils/FastByteComparisons.java
+++ b/src/java/org/apache/cassandra/utils/FastByteComparisons.java
@@ -69,6 +69,11 @@ abstract class FastByteComparisons {
      * implementation if unable to do so.
      */
     static Comparer<byte[]> getBestComparer() {
+      String arch = System.getProperty("os.arch");
+      boolean unaligned = arch.equals("i386") || arch.equals("x86")
+                    || arch.equals("amd64") || arch.equals("x86_64");
+      if (!unaligned)
+        return lexicographicalComparerJavaImpl();
       try {
         Class<?> theClass = Class.forName(UNSAFE_COMPARER_NAME);
 
@@ -229,6 +234,7 @@ abstract class FastByteComparisons {
         }
         return length1 - length2;
       }
+
     }
   }
 }


Mime
View raw message