Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@accumulo.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1478154 - in /accumulo/branches/1.5/start/src:
 main/java/org/apache/accumulo/start/classloader/vfs/
 test/java/org/apache/accumulo/start/classloader/vfs/ test/resources/
Date: Wed, 01 May 2013 19:30:57 -0000
To: commits@accumulo.apache.org
From: kturner@apache.org
Message-Id: <20130501193057.A91F523888E7@eris.apache.org>

Author: kturner
Date: Wed May  1 19:30:46 2013
New Revision: 1478154

URL: http://svn.apache.org/r1478154
Log:
ACCUMULO-1321 Created new file vfs file replicator avoid collisions, added classloader unit test that uses two jars, added code to delete replicated files and close vfs instances

Added:
    accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java
    accumulo/branches/1.5/start/src/test/resources/HelloWorld2.jar   (with props)
Modified:
    accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
    accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
    accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
    accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java

Modified: accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1478154&r1=1478153&r2=1478154&view=diff
==============================================================================
--- accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java (original)
+++ accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java Wed May  1 19:30:46 2013
@@ -64,7 +64,7 @@ public class AccumuloReloadingVFSClassLo
   public synchronized ClassLoader getClassLoader() {
     if (cl == null || cl.getParent() != parent.getClassLoader()) {
       try {
-        vfs = AccumuloVFSClassLoader.generateVfs(true);
+        vfs = AccumuloVFSClassLoader.generateVfs();
         files = AccumuloVFSClassLoader.resolve(vfs, uris);
 
         log.debug("Rebuilding dynamic classloader using files- " + stringify(files));

Modified: accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1478154&r1=1478153&r2=1478154&view=diff
==============================================================================
--- accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java (original)
+++ accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java Wed May  1 19:30:46 2013
@@ -18,10 +18,12 @@ package org.apache.accumulo.start.classl
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.ref.WeakReference;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 
 import org.apache.accumulo.start.classloader.AccumuloClassLoader;
 import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
@@ -31,7 +33,6 @@ import org.apache.commons.vfs2.FileSyste
 import org.apache.commons.vfs2.FileSystemManager;
 import org.apache.commons.vfs2.FileType;
 import org.apache.commons.vfs2.cache.SoftRefFilesCache;
-import org.apache.commons.vfs2.impl.DefaultFileReplicator;
 import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
 import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
@@ -70,6 +71,9 @@ public class AccumuloVFSClassLoader {
     
   }
   
+  private static List<WeakReference<DefaultFileSystemManager>> vfsInstances = Collections
+      .synchronizedList(new ArrayList<WeakReference<DefaultFileSystemManager>>());
+
   public static final String DYNAMIC_CLASSPATH_PROPERTY_NAME = "general.dynamic.classpaths";
   
   public static final String DEFAULT_DYNAMIC_CLASSPATH_VALUE = "$ACCUMULO_HOME/lib/ext/[^.].*.jar\n";
@@ -80,7 +84,6 @@ public class AccumuloVFSClassLoader {
   
   public static final String VFS_CACHE_DIR = "general.vfs.cache.dir";
   
-  private static DefaultFileSystemManager vfs = null;
   private static ClassLoader parent = null;
   private static volatile ReloadingClassLoader loader = null;
   private static final Object lock = new Object();
@@ -186,7 +189,7 @@ public class AccumuloVFSClassLoader {
       return wrapper;
     
     // TODO monitor time for lib/ext was 1 sec... should this be configurable? - ACCUMULO-1301
-    return new AccumuloReloadingVFSClassLoader(dynamicCPath, vfs, wrapper, 1000, true);
+    return new AccumuloReloadingVFSClassLoader(dynamicCPath, generateVfs(), wrapper, 1000, true);
   }
   
   public static ClassLoader getClassLoader() throws IOException {
@@ -195,10 +198,9 @@ public class AccumuloVFSClassLoader {
       synchronized (lock) {
         if (null == loader) {
           
-          if (null == vfs) {
-            vfs = generateVfs(false);
-          }
           
+          FileSystemManager vfs = generateVfs();
+
           // Set up the 2nd tier class loader
           if (null == parent) {
             parent = AccumuloClassLoader.getClassLoader();
@@ -213,7 +215,7 @@ public class AccumuloVFSClassLoader {
           }
           
           // Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
-          localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, generateVfs(true), parent));
+          localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP, vfs, parent));
           loader = localLoader;
         }
       }
@@ -222,8 +224,8 @@ public class AccumuloVFSClassLoader {
     return localLoader.getClassLoader();
   }
   
-  public static FinalCloseDefaultFileSystemManager generateVfs(boolean reloading) throws FileSystemException {
-    FinalCloseDefaultFileSystemManager vfs = new FinalCloseDefaultFileSystemManager();
+  public static FileSystemManager generateVfs() throws FileSystemException {
+    DefaultFileSystemManager vfs = new FinalCloseDefaultFileSystemManager();
     vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
     vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
     vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
@@ -258,12 +260,13 @@ public class AccumuloVFSClassLoader {
     vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
     vfs.setFilesCache(new SoftRefFilesCache());
     String cacheDirPath = AccumuloClassLoader.getAccumuloString(VFS_CACHE_DIR, "");
-    File cacheDir = new File(System.getProperty("java.io.tmpdir"), (reloading? "accumulo-vfs-reloading-cache":"accumulo-vfs-cache-") + System.getProperty("user.name", "nouser"));
+    File cacheDir = new File(System.getProperty("java.io.tmpdir"), "accumulo-vfs-cache-" + System.getProperty("user.name", "nouser"));
     if (!("".equals(cacheDirPath)))
       cacheDir = new File(cacheDirPath);
-    vfs.setReplicator(new DefaultFileReplicator(cacheDir));
+    vfs.setReplicator(new UniqueFileReplicator(cacheDir));
     vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
     vfs.init();
+    vfsInstances.add(new WeakReference<DefaultFileSystemManager>(vfs));
     return vfs;
   }
   
@@ -327,7 +330,7 @@ public class AccumuloVFSClassLoader {
   public static synchronized ContextManager getContextManager() throws IOException {
     if (contextManager == null) {
       getClassLoader();
-      contextManager = new ContextManager(vfs, new ReloadingClassLoader() {
+      contextManager = new ContextManager(generateVfs(), new ReloadingClassLoader() {
         @Override
         public ClassLoader getClassLoader() {
           try {
@@ -343,7 +346,10 @@ public class AccumuloVFSClassLoader {
   }
   
   public static void close() {
-    if (null != vfs)
-      vfs.close();
+    for (WeakReference<DefaultFileSystemManager> vfsInstance : vfsInstances) {
+      DefaultFileSystemManager ref = vfsInstance.get();
+      if (ref != null)
+        ref.close();
+    }
   }
 }

Added: accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java?rev=1478154&view=auto
==============================================================================
--- accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java (added)
+++ accumulo/branches/1.5/start/src/main/java/org/apache/accumulo/start/classloader/vfs/UniqueFileReplicator.java Wed May  1 19:30:46 2013
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSelector;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.provider.FileReplicator;
+import org.apache.commons.vfs2.provider.UriParser;
+import org.apache.commons.vfs2.provider.VfsComponent;
+import org.apache.commons.vfs2.provider.VfsComponentContext;
+
+/**
+ * 
+ */
+public class UniqueFileReplicator implements VfsComponent, FileReplicator {
+  
+  private static final char[] TMP_RESERVED_CHARS = new char[] {'?', '/', '\\', ' ', '&', '"', '\'', '*', '#', ';', ':', '<', '>', '|'};
+  
+  private File tempDir;
+  private VfsComponentContext context;
+  private List<File> tmpFiles = Collections.synchronizedList(new ArrayList<File>());
+  
+  public UniqueFileReplicator(File tempDir) {
+    this.tempDir = tempDir;
+  }
+
+  @Override
+  public File replicateFile(FileObject srcFile, FileSelector selector) throws FileSystemException {
+    String baseName = srcFile.getName().getBaseName();
+    
+    try {
+      tempDir.mkdirs();
+      String safeBasename = UriParser.encode(baseName, TMP_RESERVED_CHARS).replace('%', '_');
+      File file = File.createTempFile("vfsr_", "_" + safeBasename, tempDir);
+      file.deleteOnExit();
+      
+      final FileObject destFile = context.toFileObject(file);
+      destFile.copyFrom(srcFile, selector);
+      
+      return file;
+    } catch (IOException e) {
+      throw new FileSystemException(e);
+    }
+  }
+  
+  @Override
+  public void setLogger(Log logger) {
+    // TODO Auto-generated method stub
+    
+  }
+  
+  @Override
+  public void setContext(VfsComponentContext context) {
+    this.context = context;
+  }
+  
+  @Override
+  public void init() throws FileSystemException {
+    
+  }
+  
+  @Override
+  public void close() {
+    synchronized (tmpFiles) {
+      for (File tmpFile : tmpFiles) {
+        tmpFile.delete();
+      }
+    }
+    
+    tempDir.delete();
+  }
+}

Modified: accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java?rev=1478154&r1=1478153&r2=1478154&view=diff
==============================================================================
--- accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java (original)
+++ accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java Wed May  1 19:30:46 2013
@@ -21,7 +21,7 @@ import java.io.File;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.FileSystemManager;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -34,7 +34,7 @@ import org.junit.rules.TemporaryFolder;
 public class AccumuloReloadingVFSClassLoaderTest {
 
   private TemporaryFolder folder1 = new TemporaryFolder();
-  private DefaultFileSystemManager vfs;
+  private FileSystemManager vfs;
 
   @Before
   public void setup() throws Exception {
@@ -120,6 +120,60 @@ public class AccumuloReloadingVFSClassLo
     arvcl.close();
   }
   
+
+  @Test
+  public void testModifiedClass() throws Exception {
+
+    FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
+    FileObject[] dirContents = testDir.getChildren();
+    
+    AccumuloReloadingVFSClassLoader arvcl = new AccumuloReloadingVFSClassLoader(folder1.getRoot().toURI().toString(), vfs, new ReloadingClassLoader() {
+      @Override
+      public ClassLoader getClassLoader() {
+        return ClassLoader.getSystemClassLoader();
+      }
+    }, 1000, true);
+    
+    FileObject[] files = ((VFSClassLoader) arvcl.getClassLoader()).getFileObjects();
+    Assert.assertArrayEquals(createFileSystems(dirContents), files);
+    
+    ClassLoader loader1 = arvcl.getClassLoader();
+    Class<?> clazz1 = loader1.loadClass("test.HelloWorld");
+    Object o1 = clazz1.newInstance();
+    Assert.assertEquals("Hello World!", o1.toString());
+    
+    // Check that the class is the same before the update
+    Class<?> clazz1_5 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+    Assert.assertEquals(clazz1, clazz1_5);
+    
+    // java does aggressive caching of jar files. When using java code to read jar files that are created in the same second, it will only see the first jar
+    // file
+    Thread.sleep(1000);
+
+    new File(folder1.getRoot(), "HelloWorld.jar").delete();
+
+    // Update the class
+    FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld2.jar"), folder1.newFile("HelloWorld.jar"));
+    
+    // Wait for the monitor to notice
+    Thread.sleep(2000);
+    
+
+    Class<?> clazz2 = arvcl.getClassLoader().loadClass("test.HelloWorld");
+    Object o2 = clazz2.newInstance();
+    Assert.assertEquals("Hallo Welt", o2.toString());
+    
+    // This is false because they are loaded by a different classloader
+    Assert.assertFalse(clazz1.equals(clazz2));
+    Assert.assertFalse(o1.equals(o2));
+    
+    Class<?> clazz3 = loader1.loadClass("test.HelloWorld");
+    Object o3 = clazz3.newInstance();
+    Assert.assertEquals("Hello World!", o3.toString());
+
+    arvcl.close();
+  }
+
   @After
   public void tearDown() throws Exception {
     folder1.delete();

Modified: accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java?rev=1478154&r1=1478153&r2=1478154&view=diff
==============================================================================
--- accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java (original)
+++ accumulo/branches/1.5/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java Wed May  1 19:30:46 2013
@@ -22,16 +22,10 @@ import java.util.HashSet;
 
 import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
 import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextsConfig;
-import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.vfs2.CacheStrategy;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.cache.DefaultFilesCache;
-import org.apache.commons.vfs2.cache.SoftRefFilesCache;
-import org.apache.commons.vfs2.impl.DefaultFileReplicator;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
-import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
+import org.apache.commons.vfs2.FileSystemManager;
 import org.apache.commons.vfs2.impl.VFSClassLoader;
 import org.junit.After;
 import org.junit.Assert;
@@ -43,55 +37,16 @@ public class ContextManagerTest {
   
   private TemporaryFolder folder1 = new TemporaryFolder();
   private TemporaryFolder folder2 = new TemporaryFolder();
-  private DefaultFileSystemManager vfs;
+  private FileSystemManager vfs;
   private URI uri1;
   private URI uri2;
   
-  static DefaultFileSystemManager getVFS() {
-    DefaultFileSystemManager vfs = new DefaultFileSystemManager();
+  static FileSystemManager getVFS() {
     try {
-      vfs.setFilesCache(new DefaultFilesCache());
-      vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
-      vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
-      vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
-      vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
-      vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
-      vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
-      vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
-      vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
-      vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
-      vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
-      vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
-      vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
-      vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
-      vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
-      vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
-      vfs.addProvider("hdfs", new HdfsFileProvider());
-      vfs.addExtensionMap("jar", "jar");
-      vfs.addExtensionMap("zip", "zip");
-      vfs.addExtensionMap("gz", "gz");
-      vfs.addExtensionMap("tar", "tar");
-      vfs.addExtensionMap("tbz2", "tar");
-      vfs.addExtensionMap("tgz", "tar");
-      vfs.addExtensionMap("bz2", "bz2");
-      vfs.addMimeTypeMap("application/x-tar", "tar");
-      vfs.addMimeTypeMap("application/x-gzip", "gz");
-      vfs.addMimeTypeMap("application/zip", "zip");
-      vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
-      vfs.setFilesCache(new SoftRefFilesCache());
-      vfs.setReplicator(new DefaultFileReplicator());
-      vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
-      vfs.init();
+      return AccumuloVFSClassLoader.generateVfs();
     } catch (FileSystemException e) {
       throw new RuntimeException("Error setting up VFS", e);
     }
-    
-    return vfs;
   }
 
   @Before

Added: accumulo/branches/1.5/start/src/test/resources/HelloWorld2.jar
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/start/src/test/resources/HelloWorld2.jar?rev=1478154&view=auto
==============================================================================
Binary file - no diff available.

Propchange: accumulo/branches/1.5/start/src/test/resources/HelloWorld2.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream