accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1478165 - in /accumulo/trunk: ./ assemble/ core/ examples/ fate/src/main/java/org/apache/accumulo/fate/ fate/src/main/java/org/apache/accumulo/fate/zookeeper/ server/ src/ start/src/main/java/org/apache/accumulo/start/classloader/vfs/ star...
Date Wed, 01 May 2013 19:48:30 GMT
Author: kturner
Date: Wed May  1 19:48:23 2013
New Revision: 1478165

URL: http://svn.apache.org/r1478165
Log:
ACCUMULO-1321 Created new file vfs file replicator avoid collisions, added classloader unit
test that uses two jars, added code to delete replicated files and close vfs instances

Added:
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java
      - copied unchanged from r1478154, accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java
    accumulo/trunk/start/src/test/resources/HelloWorld2.jar
      - copied unchanged from r1478154, accumulo/branches/1.5/start/src/test/resources/HelloWorld2.jar
Modified:
    accumulo/trunk/   (props changed)
    accumulo/trunk/assemble/   (props changed)
    accumulo/trunk/core/   (props changed)
    accumulo/trunk/examples/   (props changed)
    accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java   (props changed)
    accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java 
 (props changed)
    accumulo/trunk/pom.xml   (props changed)
    accumulo/trunk/server/   (props changed)
    accumulo/trunk/src/   (props changed)
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
    accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
    accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java

Propchange: accumulo/trunk/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5:r1478154

Propchange: accumulo/trunk/assemble/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/assemble:r1478154

Propchange: accumulo/trunk/core/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/core:r1478154

Propchange: accumulo/trunk/examples/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/examples:r1478154

Propchange: accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:r1478154

Propchange: accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java:r1478154

Propchange: accumulo/trunk/pom.xml
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/pom.xml:r1478154

Propchange: accumulo/trunk/server/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/server:r1478154

Propchange: accumulo/trunk/src/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/src:r1478154

Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1478165&r1=1478164&r2=1478165&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
(original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
Wed May  1 19:48:23 2013
@@ -64,7 +64,7 @@ public class AccumuloReloadingVFSClassLo
   public synchronized ClassLoader getClassLoader() {
     if (cl == null || cl.getParent() != parent.getClassLoader()) {
       try {
-        vfs = AccumuloVFSClassLoader.generateVfs(true);
+        vfs = AccumuloVFSClassLoader.generateVfs();
         files = AccumuloVFSClassLoader.resolve(vfs, uris);
 
         log.debug("Rebuilding dynamic classloader using files- " + stringify(files));

Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1478165&r1=1478164&r2=1478165&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
(original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
Wed May  1 19:48:23 2013
@@ -18,10 +18,12 @@ package org.apache.accumulo.start.classl
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.ref.WeakReference;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 
 import org.apache.accumulo.start.classloader.AccumuloClassLoader;
 import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
@@ -31,7 +33,6 @@ import org.apache.commons.vfs2.FileSyste
 import org.apache.commons.vfs2.FileSystemManager;
 import org.apache.commons.vfs2.FileType;
 import org.apache.commons.vfs2.cache.SoftRefFilesCache;
-import org.apache.commons.vfs2.impl.DefaultFileReplicator;
 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
 import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
@@ -70,6 +71,9 @@ public class AccumuloVFSClassLoader {
     
   }
   
+  private static List<WeakReference<DefaultFileSystemManager>> vfsInstances =
Collections
+      .synchronizedList(new ArrayList<WeakReference<DefaultFileSystemManager>>());
+
   public static final String DYNAMIC_CLASSPATH_PROPERTY_NAME = "general.dynamic.classpaths";
   
   public static final String DEFAULT_DYNAMIC_CLASSPATH_VALUE = "$ACCUMULO_HOME/lib/ext/[^.].*.jar\n";
@@ -80,7 +84,6 @@ public class AccumuloVFSClassLoader {
   
   public static final String VFS_CACHE_DIR = "general.vfs.cache.dir";
   
-  private static DefaultFileSystemManager vfs = null;
   private static ClassLoader parent = null;
   private static volatile ReloadingClassLoader loader = null;
   private static final Object lock = new Object();
@@ -186,7 +189,7 @@ public class AccumuloVFSClassLoader {
       return wrapper;
     
     // TODO monitor time for lib/ext was 1 sec... should this be configurable? - ACCUMULO-1301
-    return new AccumuloReloadingVFSClassLoader(dynamicCPath, vfs, wrapper, 1000, true);
+    return new AccumuloReloadingVFSClassLoader(dynamicCPath, generateVfs(), wrapper, 1000,
true);
   }
   
   public static ClassLoader getClassLoader() throws IOException {
@@ -195,10 +198,9 @@ public class AccumuloVFSClassLoader {
       synchronized (lock) {
         if (null == loader) {
           
-          if (null == vfs) {
-            vfs = generateVfs(false);
-          }
           
+          FileSystemManager vfs = generateVfs();
+
           // Set up the 2nd tier class loader
           if (null == parent) {
             parent = AccumuloClassLoader.getClassLoader();
@@ -213,7 +215,7 @@ public class AccumuloVFSClassLoader {
           }
           
           // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
-          localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, generateVfs(true),
parent));
+          localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent));
           loader = localLoader;
         }
       }
@@ -222,8 +224,8 @@ public class AccumuloVFSClassLoader {
     return localLoader.getClassLoader();
   }
   
-  public static FinalCloseDefaultFileSystemManager generateVfs(boolean reloading) throws
FileSystemException {
-    FinalCloseDefaultFileSystemManager vfs = new FinalCloseDefaultFileSystemManager();
+  public static FileSystemManager generateVfs() throws FileSystemException {
+    DefaultFileSystemManager vfs = new FinalCloseDefaultFileSystemManager();
     vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
     vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
     vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
@@ -258,12 +260,13 @@ public class AccumuloVFSClassLoader {
     vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
     vfs.setFilesCache(new SoftRefFilesCache());
     String cacheDirPath = AccumuloClassLoader.getAccumuloString(VFS_CACHE_DIR, "");
-    File cacheDir = new File(System.getProperty("java.io.tmpdir"), (reloading? "accumulo-vfs-reloading-cache":"accumulo-vfs-cache-")
+ System.getProperty("user.name", "nouser"));
+    File cacheDir = new File(System.getProperty("java.io.tmpdir"), "accumulo-vfs-cache-"
+ System.getProperty("user.name", "nouser"));
     if (!("".equals(cacheDirPath)))
       cacheDir = new File(cacheDirPath);
-    vfs.setReplicator(new DefaultFileReplicator(cacheDir));
+    vfs.setReplicator(new UniqueFileReplicator(cacheDir));
     vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
     vfs.init();
+    vfsInstances.add(new WeakReference<DefaultFileSystemManager>(vfs));
     return vfs;
   }
   
@@ -327,7 +330,7 @@ public class AccumuloVFSClassLoader {
   public static synchronized ContextManager getContextManager() throws IOException {
     if (contextManager == null) {
       getClassLoader();
-      contextManager = new ContextManager(vfs, new ReloadingClassLoader() {
+      contextManager = new ContextManager(generateVfs(), new ReloadingClassLoader() {
         @Override
         public ClassLoader getClassLoader() {
           try {
@@ -343,7 +346,10 @@ public class AccumuloVFSClassLoader {
   }
   
   public static void close() {
-    if (null != vfs)
-      vfs.close();
+    for (WeakReference<DefaultFileSystemManager> vfsInstance : vfsInstances) {
+      DefaultFileSystemManager ref = vfsInstance.get();
+      if (ref != null)
+        ref.close();
+    }
   }
 }

Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java?rev=1478165&r1=1478164&r2=1478165&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
(original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
Wed May  1 19:48:23 2013
@@ -21,7 +21,7 @@ import java.io.File;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.FileSystemManager;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -34,7 +34,7 @@ import org.junit.rules.TemporaryFolder;
 public class AccumuloReloadingVFSClassLoaderTest {
 
   private TemporaryFolder folder1 = new TemporaryFolder();
-  private DefaultFileSystemManager vfs;
+  private FileSystemManager vfs;
 
   @Before
   public void setup() throws Exception {
@@ -120,6 +120,60 @@ public class AccumuloReloadingVFSClassLo
     arvcl.close();
   }
   
+
+  @Test
+  public void testModifiedClass() throws Exception {
+
+    FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
+    FileObject[] dirContents = testDir.getChildren();
+    
+    AccumuloReloadingVFSClassLoader arvcl = new AccumuloReloadingVFSClassLoader(folder1.getRoot().toURI().toString(),
vfs, new ReloadingClassLoader() {
+      @Override
+      public ClassLoader getClassLoader() {
+        return ClassLoader.getSystemClassLoader();
+      }
+    }, 1000, true);
+    
+    FileObject[] files = ((VFSClassLoader) arvcl.getClassLoader()).getFileObjects();
+    Assert.assertArrayEquals(createFileSystems(dirContents), files);
+    
+    ClassLoader loader1 = arvcl.getClassLoader();
+    Class<?> clazz1 = loader1.loadClass("test.HelloWorld");
+    Object o1 = clazz1.newInstance();
+    Assert.assertEquals("Hello World!", o1.toString());
+    
+    // Check that the class is the same before the update
+    Class<?> clazz1_5 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+    Assert.assertEquals(clazz1, clazz1_5);
+    
+    // java does aggressive caching of jar files. When using java code to read jar files
that are created in the same second, it will only see the first jar
+    // file
+    Thread.sleep(1000);
+
+    new File(folder1.getRoot(), "HelloWorld.jar").delete();
+
+    // Update the class
+    FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld2.jar"), folder1.newFile("HelloWorld.jar"));
+    
+    // Wait for the monitor to notice
+    Thread.sleep(2000);
+    
+
+    Class<?> clazz2 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+    Object o2 = clazz2.newInstance();
+    Assert.assertEquals("Hallo Welt", o2.toString());
+    
+    // This is false because they are loaded by a different classloader
+    Assert.assertFalse(clazz1.equals(clazz2));
+    Assert.assertFalse(o1.equals(o2));
+    
+    Class<?> clazz3 = loader1.loadClass("test.HelloWorld");
+    Object o3 = clazz3.newInstance();
+    Assert.assertEquals("Hello World!", o3.toString());
+
+    arvcl.close();
+  }
+
   @After
   public void tearDown() throws Exception {
     folder1.delete();

Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java?rev=1478165&r1=1478164&r2=1478165&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
(original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
Wed May  1 19:48:23 2013
@@ -22,16 +22,10 @@ import java.util.HashSet;
 
 import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
 import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextsConfig;
-import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.vfs2.CacheStrategy;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.cache.DefaultFilesCache;
-import org.apache.commons.vfs2.cache.SoftRefFilesCache;
-import org.apache.commons.vfs2.impl.DefaultFileReplicator;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
-import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
+import org.apache.commons.vfs2.FileSystemManager;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
 import org.junit.After;
 import org.junit.Assert;
@@ -43,55 +37,16 @@ public class ContextManagerTest {
   
   private TemporaryFolder folder1 = new TemporaryFolder();
   private TemporaryFolder folder2 = new TemporaryFolder();
-  private DefaultFileSystemManager vfs;
+  private FileSystemManager vfs;
   private URI uri1;
   private URI uri2;
   
-  static DefaultFileSystemManager getVFS() {
-    DefaultFileSystemManager vfs = new DefaultFileSystemManager();
+  static FileSystemManager getVFS() {
     try {
-      vfs.setFilesCache(new DefaultFilesCache());
-      vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
-      vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
-      vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
-      vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
-      vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
-      vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
-      vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
-      vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
-      vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
-      vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
-      vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
-      vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
-      vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
-      vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
-      vfs.addProvider("hdfs", new HdfsFileProvider());
-      vfs.addExtensionMap("jar", "jar");
-      vfs.addExtensionMap("zip", "zip");
-      vfs.addExtensionMap("gz", "gz");
-      vfs.addExtensionMap("tar", "tar");
-      vfs.addExtensionMap("tbz2", "tar");
-      vfs.addExtensionMap("tgz", "tar");
-      vfs.addExtensionMap("bz2", "bz2");
-      vfs.addMimeTypeMap("application/x-tar", "tar");
-      vfs.addMimeTypeMap("application/x-gzip", "gz");
-      vfs.addMimeTypeMap("application/zip", "zip");
-      vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
-      vfs.setFilesCache(new SoftRefFilesCache());
-      vfs.setReplicator(new DefaultFileReplicator());
-      vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
-      vfs.init();
+      return AccumuloVFSClassLoader.generateVfs();
     } catch (FileSystemException e) {
       throw new RuntimeException("Error setting up VFS", e);
     }
-    
-    return vfs;
   }
 
   @Before



Mime
View raw message