activemq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chir...@apache.org
Subject svn commit: r1232161 - in /activemq/activemq-apollo/trunk: ./ apollo-leveldb/ apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/ apollo-util/ apollo-util/src/main/scala/org/apache/activemq/apollo/util/ apollo-util/sr...
Date Mon, 16 Jan 2012 20:57:20 GMT
Author: chirino
Date: Mon Jan 16 20:57:20 2012
New Revision: 1232161

URL: http://svn.apache.org/viewvc?rev=1232161&view=rev
Log:
Switch to the lastest leveldb snapshot as that helps resolve lots of issues on windows.  Also
fallback to using JNA based hardlink system calls as the speeds up links on windows.

Added:
    activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32JnaLibrary.java
  (contents, props changed)
      - copied, changed from r1232113, activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
    activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32Library.java
  (contents, props changed)
      - copied, changed from r1232113, activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
Modified:
    activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml   (contents, props changed)
    activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
    activemq/activemq-apollo/trunk/apollo-util/pom.xml   (contents, props changed)
    activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/IOHelper.java
  (contents, props changed)
    activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
  (contents, props changed)
    activemq/activemq-apollo/trunk/pom.xml   (contents, props changed)

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml?rev=1232161&r1=1232160&r2=1232161&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml Mon Jan 16 20:57:20 2012
@@ -45,27 +45,27 @@
     <dependency>
       <groupId>org.iq80.leveldb</groupId>
       <artifactId>leveldb-api</artifactId>
-      <version>0.1</version>
+      <version>${leveldb-api-version}</version>
     </dependency>
     <dependency>
       <groupId>org.iq80.leveldb</groupId>
       <artifactId>leveldb</artifactId>
-      <version>0.1</version>
+      <version>${leveldb-version}</version>
     </dependency>
     <dependency>
       <groupId>org.fusesource.leveldbjni</groupId>
       <artifactId>leveldbjni-osx</artifactId>
-      <version>1.1</version>
+      <version>${leveldbjni-version}</version>
     </dependency>
     <dependency>
       <groupId>org.fusesource.leveldbjni</groupId>
       <artifactId>leveldbjni-linux32</artifactId>
-      <version>1.1</version>
+      <version>${leveldbjni-version}</version>
     </dependency>
     <dependency>
       <groupId>org.fusesource.leveldbjni</groupId>
       <artifactId>leveldbjni-linux64</artifactId>
-      <version>1.1</version>
+      <version>${leveldbjni-version}</version>
     </dependency>
 
 

Propchange: activemq/activemq-apollo/trunk/apollo-leveldb/pom.xml
------------------------------------------------------------------------------
    svn:executable = *

Modified: activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala?rev=1232161&r1=1232160&r2=1232161&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
(original)
+++ activemq/activemq-apollo/trunk/apollo-leveldb/src/main/scala/org/apache/activemq/apollo/broker/store/leveldb/leveldb/LevelDBClient.scala
Mon Jan 16 20:57:20 2012
@@ -94,7 +94,6 @@ object LevelDBClient extends Log {
   }
 
   val on_windows = System.getProperty("os.name").toLowerCase().startsWith("windows")
-
   var link_strategy = 0
   def link(source:File, target:File):Unit = {
     link_strategy match {
@@ -108,11 +107,23 @@ object LevelDBClient extends Log {
           case e:Throwable =>
             // Fallback.. to a slower impl..
             debug("Native link system call not available")
-            link_strategy = 5
+            link_strategy = 2
             link(source, target)
         }
 
       // TODO: consider implementing a case which does the native system call using JNA
+      case 2 =>
+        // Next try JNA (might not be in classpath)
+        try {
+          IOHelper.hardlink(source, target)
+        } catch {
+          case e:IOException => throw e
+          case e:Throwable =>
+            // Fallback.. to a slower impl..
+            debug("JNA based hard link system call not available")
+            link_strategy = 5
+            link(source, target)
+        }
 
       case 5 =>
         // Next we try to do the link by executing an
@@ -135,7 +146,7 @@ object LevelDBClient extends Log {
                 // TODO: we might want to look at the out/err to see why it failed
                 // to avoid falling back to the slower strategy.
                 debug("ln OS command not available either")
-                link_strategy = 2
+                link_strategy = 10
                 link(source, target)
             }
           }

Modified: activemq/activemq-apollo/trunk/apollo-util/pom.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-util/pom.xml?rev=1232161&r1=1232160&r2=1232161&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-util/pom.xml (original)
+++ activemq/activemq-apollo/trunk/apollo-util/pom.xml Mon Jan 16 20:57:20 2012
@@ -65,7 +65,7 @@
     <dependency>
       <groupId>net.java.dev.jna</groupId>
       <artifactId>jna</artifactId>
-      <version>3.1.0</version>
+      <version>3.4.0</version>
       <optional>true</optional>
     </dependency>
 

Propchange: activemq/activemq-apollo/trunk/apollo-util/pom.xml
------------------------------------------------------------------------------
    svn:executable = *

Modified: activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/IOHelper.java
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/IOHelper.java?rev=1232161&r1=1232160&r2=1232161&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/IOHelper.java
(original)
+++ activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/IOHelper.java
Mon Jan 16 20:57:20 2012
@@ -20,6 +20,7 @@ import java.io.*;
 import java.lang.reflect.Field;
 
 import org.apache.activemq.apollo.util.os.CLibrary;
+import org.apache.activemq.apollo.util.os.Kernel32Library;
 import org.fusesource.hawtbuf.ByteArrayOutputStream;
 import org.fusesource.hawtbuf.HexSupport;
 
@@ -254,13 +255,15 @@ public final class IOHelper {
         }
     }
     
-	public interface IOStrategy {
+	public interface SyncStrategy {
 		void sync(FileDescriptor fdo) throws IOException;
-	}	
-	
-	static final IOStrategy IO_STRATEGY = createIOStrategy();
-	
-	private static IOStrategy createIOStrategy() {
+	}
+    static final SyncStrategy SYNC_STRATEGY = createSyncStrategy();
+    static public void sync(FileDescriptor fd) throws IOException {
+        SYNC_STRATEGY.sync(fd);
+    }
+
+	private static SyncStrategy createSyncStrategy() {
 		
 		// On OS X, the fsync system call does not fully flush the hardware buffers.. 
 		// to do that you have to do an fcntl call, and the only way to do that is to
@@ -276,7 +279,7 @@ public final class IOHelper {
 				field.setAccessible(true);
 				// Try to dynamically load the JNA impl of the CLibrary interface..
 				final CLibrary lib = getCLibrary();
-				return new IOStrategy() {
+				return new SyncStrategy() {
 					static final int F_FULLFSYNC = 51;	
 					public void sync(FileDescriptor fd) throws IOException {
 						try {
@@ -286,29 +289,130 @@ public final class IOHelper {
 							throw IOExceptionSupport.create(e);
 						}
 					}
-				};
+
+                    public void hardlink(File source, File target) throws IOException {
+                        int rc = lib.link(source.getCanonicalPath(), target.getCanonicalPath());
+                        if( rc != 0 ){
+                            throw new IOException("Hard link failed with result code="+rc);
+                        }
+                    }
+                };
 			} catch (Throwable ignore) {
 				// Perhaps we should issue a warning here so folks know that 
 				// the disk syncs are not going to be of very good quality.
 			}
-		}
-		
-		return new IOStrategy() {
+		} else if( os.toLowerCase().startsWith("windows") ) {
+            // We will gracefully fall back to default JDK file sync behavior
+            // if the JNA library is not in the path, and we can't set the
+            // FileDescriptor.fd field accessible.
+            try {
+                final Kernel32Library lib = getKernel32Library();
+                return new SyncStrategy() {
+                    public void sync(FileDescriptor fd) throws IOException {
+                        fd.sync();
+                    }
+                    public void hardlink(File source, File target) throws IOException {
+                        int rc = lib.CreateHardLink(target.getCanonicalPath(), source.getCanonicalPath(),
0);
+                        if( rc == 0 ){
+                            throw new IOException("Hard link failed with result code="+lib.GetLastError());
+                        }
+                    }
+                };
+            } catch (Throwable ignore) {
+                // Perhaps we should issue a warning here so folks know that
+                // the disk syncs are not going to be of very good quality.
+            }
+        }
+
+
+        // We will gracefully fall back to default JDK file sync behavior
+        // if the JNA library is not in the path, and we can't set the
+        // FileDescriptor.fd field accessible.
+        try {
+            final CLibrary lib = getCLibrary();
+            return new SyncStrategy() {
+                public void sync(FileDescriptor fd) throws IOException {
+                    fd.sync();
+                }
+
+                public void hardlink(File source, File target) throws IOException {
+                    int rc = lib.link(source.getCanonicalPath(), target.getCanonicalPath());
+                    if( rc != 0 ){
+                        throw new IOException("Hard link failed with result code="+rc);
+                    }
+                }
+            };
+        } catch (Throwable ignore) {
+            // Perhaps we should issue a warning here so folks know that
+            // the disk syncs are not going to be of very good quality.
+        }
+
+		return new SyncStrategy() {
 			public void sync(FileDescriptor fd) throws IOException {
 				fd.sync();
 			}
-		};
+
+            public void hardlink(File source, File target) throws IOException {
+
+            }
+        };
 	}
 
+    public interface HardLinkStrategy {
+        void hardlink(File source, File target) throws IOException;
+    }
+    static final HardLinkStrategy HARD_LINK_STRATEGY = createHardLinkStrategy();
+    static public void hardlink(File source, File target) throws IOException {
+        if(HARD_LINK_STRATEGY==null)
+            throw new UnsupportedOperationException();
+        HARD_LINK_STRATEGY.hardlink(source, target);
+    }
+
+    private static HardLinkStrategy createHardLinkStrategy() {
+
+        String os = System.getProperty("os.name");
+        if( os.toLowerCase().startsWith("windows") ) {
+            try {
+                final Kernel32Library lib = getKernel32Library();
+                return new HardLinkStrategy() {
+                    public void hardlink(File source, File target) throws IOException {
+                        int rc = lib.CreateHardLink(target.getCanonicalPath(), source.getCanonicalPath(),
0);
+                        if( rc == 0 ){
+                            throw new IOException("Hard link failed with result code="+lib.GetLastError());
+                        }
+                    }
+                };
+            } catch (Throwable ignore) {
+            }
+        }
+
+        try {
+            final CLibrary lib = getCLibrary();
+            return new HardLinkStrategy() {
+                public void hardlink(File source, File target) throws IOException {
+                    int rc = lib.link(source.getCanonicalPath(), target.getCanonicalPath());
+                    if( rc != 0 ){
+                        throw new IOException("Hard link failed with result code="+rc);
+                    }
+                }
+            };
+        } catch (Throwable ignore) {
+        }
+        return null;
+    }
+
 	@SuppressWarnings("unchecked")
 	public static CLibrary getCLibrary() throws ClassNotFoundException, IllegalAccessException,
NoSuchFieldException {
-		Class clazz = IOHelper.class.getClassLoader().loadClass("org.apache.activemq.util.os.JnaCLibrary");
+		Class clazz = IOHelper.class.getClassLoader().loadClass("org.apache.activemq.apollo..util.os.JnaCLibrary");
 		final CLibrary lib = (CLibrary) clazz.getField("INSTANCE").get(null);
 		return lib;
 	}
-	
-	static public void sync(FileDescriptor fd) throws IOException {
-		IO_STRATEGY.sync(fd);
-	}
+
+    @SuppressWarnings("unchecked")
+    public static Kernel32Library getKernel32Library() throws ClassNotFoundException, IllegalAccessException,
NoSuchFieldException {
+        Class clazz = IOHelper.class.getClassLoader().loadClass("org.apache.activemq.apollo.util.os.Kernel32JnaLibrary");
+        final Kernel32Library lib = (Kernel32Library) clazz.getField("INSTANCE").get(null);
+        return lib;
+    }
 
 }

Propchange: activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/IOHelper.java
------------------------------------------------------------------------------
    svn:executable = *

Modified: activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java?rev=1232161&r1=1232160&r2=1232161&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
(original)
+++ activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
Mon Jan 16 20:57:20 2012
@@ -18,4 +18,5 @@ package org.apache.activemq.apollo.util.
 
 public interface CLibrary {
     void fcntl(int fd, int cmd, Object... args);
+    int link(String source, String target);
 }
\ No newline at end of file

Propchange: activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
------------------------------------------------------------------------------
    svn:executable = *

Copied: activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32JnaLibrary.java
(from r1232113, activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java)
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32JnaLibrary.java?p2=activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32JnaLibrary.java&p1=activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java&r1=1232113&r2=1232161&rev=1232161&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
(original)
+++ activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32JnaLibrary.java
Mon Jan 16 20:57:20 2012
@@ -1,21 +1,31 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.activemq.apollo.util.os;
-
-public interface CLibrary {
-    void fcntl(int fd, int cmd, Object... args);
-}
\ No newline at end of file
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.activemq.apollo.util.os;
+
+import com.sun.jna.Library;
+import com.sun.jna.Native;
+import com.sun.jna.win32.W32APIOptions;
+
+/**
+ * <p>
+ * </p>
+ *
+ * @author <a href="http://hiramchirino.com">Hiram Chirino</a>
+ */
+public interface Kernel32JnaLibrary extends Library, Kernel32Library {
+    Kernel32JnaLibrary INSTANCE = (Kernel32JnaLibrary) Native.loadLibrary("kernel32", Kernel32JnaLibrary.class,
W32APIOptions.UNICODE_OPTIONS);
+}

Propchange: activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32JnaLibrary.java
------------------------------------------------------------------------------
    svn:executable = *

Copied: activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32Library.java
(from r1232113, activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java)
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32Library.java?p2=activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32Library.java&p1=activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java&r1=1232113&r2=1232161&rev=1232161&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/CLibrary.java
(original)
+++ activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32Library.java
Mon Jan 16 20:57:20 2012
@@ -16,6 +16,13 @@
  */
 package org.apache.activemq.apollo.util.os;
 
-public interface CLibrary {
-    void fcntl(int fd, int cmd, Object... args);
+/**
+ * <p>
+ * </p>
+ *
+ * @author <a href="http://hiramchirino.com">Hiram Chirino</a>
+ */
+public interface Kernel32Library {
+    public int GetLastError();
+    public int CreateHardLink(String target, String source, long reserved);
 }
\ No newline at end of file

Propchange: activemq/activemq-apollo/trunk/apollo-util/src/main/scala/org/apache/activemq/apollo/util/os/Kernel32Library.java
------------------------------------------------------------------------------
    svn:executable = *

Modified: activemq/activemq-apollo/trunk/pom.xml
URL: http://svn.apache.org/viewvc/activemq/activemq-apollo/trunk/pom.xml?rev=1232161&r1=1232160&r2=1232161&view=diff
==============================================================================
--- activemq/activemq-apollo/trunk/pom.xml (original)
+++ activemq/activemq-apollo/trunk/pom.xml Mon Jan 16 20:57:20 2012
@@ -101,6 +101,11 @@
     
     <jdbm-version>2.0.1</jdbm-version>
     <bdb-version>4.1.10</bdb-version>
+
+    <leveldb-api-version>0.1</leveldb-api-version>
+    <leveldb-version>0.2-SNAPSHOT</leveldb-version>
+    <leveldbjni-version>1.1</leveldbjni-version>
+
     <jasypt-version>1.6</jasypt-version>
 
     <scalate-version>1.5.3</scalate-version>

Propchange: activemq/activemq-apollo/trunk/pom.xml
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message