Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2D924EC5A for ; Thu, 10 Jan 2013 19:47:46 +0000 (UTC) Received: (qmail 45449 invoked by uid 500); 10 Jan 2013 19:47:46 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 45418 invoked by uid 500); 10 Jan 2013 19:47:46 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 45411 invoked by uid 99); 10 Jan 2013 19:47:46 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Jan 2013 19:47:46 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Jan 2013 19:47:41 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id AA831238899C; Thu, 10 Jan 2013 19:47:20 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1431579 - in /accumulo/trunk/start/src/test: java/org/apache/accumulo/start/classloader/vfs/ resources/default/ resources/disabled/ Date: Thu, 10 Jan 2013 19:47:20 -0000 To: commits@accumulo.apache.org From: kturner@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130110194720.AA831238899C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kturner Date: Thu Jan 10 19:47:20 2013 New Revision: 1431579 URL: http://svn.apache.org/viewvc?rev=1431579&view=rev Log: ACCUMULO-866 made some of the new VFS classloader test use local filesystem instead of HDFS. This took about 20 to 30 seconds of the time it takes to package accumulo-start on my machine. Removed: accumulo/trunk/start/src/test/resources/default/ accumulo/trunk/start/src/test/resources/disabled/ Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java?rev=1431579&r1=1431578&r2=1431579&view=diff ============================================================================== --- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java (original) +++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java Thu Jan 10 19:47:20 2013 @@ -16,40 +16,34 @@ */ package org.apache.accumulo.start.classloader.vfs; -import java.net.URL; +import java.io.File; -import org.apache.accumulo.test.AccumuloDFSBase; +import org.apache.commons.io.FileUtils; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.impl.DefaultFileSystemManager; import org.apache.commons.vfs2.impl.VFSClassLoader; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.rules.TemporaryFolder; -public class AccumuloReloadingVFSClassLoaderTest extends AccumuloDFSBase { +public class AccumuloReloadingVFSClassLoaderTest { - private static final Path TEST_DIR = new Path(HDFS_URI + "/test-dir"); - - private FileSystem hdfs = null; + private TemporaryFolder folder1 = new TemporaryFolder(); + private DefaultFileSystemManager vfs; @Before public void setup() throws Exception { Logger.getRootLogger().setLevel(Level.ERROR); - this.hdfs = cluster.getFileSystem(); - this.hdfs.mkdirs(TEST_DIR); - - //Copy jar file to TEST_DIR - URL jarPath = this.getClass().getResource("/HelloWorld.jar"); - Path src = new Path(jarPath.toURI().toString()); - Path dst = new Path(TEST_DIR, src.getName()); - this.hdfs.copyFromLocalFile(src, dst); + vfs = ContextManagerTest.getVFS(); + folder1.create(); + FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"), folder1.newFile("HelloWorld.jar")); } FileObject[] createFileSystems(FileObject[] fos) throws FileSystemException { @@ -66,10 +60,10 @@ public class AccumuloReloadingVFSClassLo @Test public void testConstructor() throws Exception { - FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString()); + FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString()); FileObject[] dirContents = testDir.getChildren(); - AccumuloReloadingVFSClassLoader arvcl = new AccumuloReloadingVFSClassLoader(TEST_DIR.toUri().toString(), vfs, new ReloadingClassLoader() { + AccumuloReloadingVFSClassLoader arvcl = new AccumuloReloadingVFSClassLoader(folder1.getRoot().toURI().toString(), vfs, new ReloadingClassLoader() { @Override public ClassLoader getClassLoader() { return ClassLoader.getSystemClassLoader(); @@ -86,10 +80,10 @@ public class AccumuloReloadingVFSClassLo @Test public void testReloading() throws Exception { - FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString()); + FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString()); FileObject[] dirContents = testDir.getChildren(); - AccumuloReloadingVFSClassLoader arvcl = new AccumuloReloadingVFSClassLoader(TEST_DIR.toUri().toString(), vfs, new ReloadingClassLoader() { + AccumuloReloadingVFSClassLoader arvcl = new AccumuloReloadingVFSClassLoader(folder1.getRoot().toURI().toString(), vfs, new ReloadingClassLoader() { @Override public ClassLoader getClassLoader() { return ClassLoader.getSystemClassLoader(); @@ -107,11 +101,10 @@ public class AccumuloReloadingVFSClassLo Class clazz1_5 = arvcl.getClassLoader().loadClass("test.HelloWorld"); Assert.assertEquals(clazz1, clazz1_5); - //Update the class - URL jarPath = this.getClass().getResource("/HelloWorld.jar"); - Path src = new Path(jarPath.toURI().toString()); - Path dst = new Path(TEST_DIR, "HelloWorld.jar"); - this.hdfs.copyFromLocalFile(src, dst); + new File(folder1.getRoot(), "HelloWorld.jar").delete(); + + // Update the class + FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"), folder1.newFile("HelloWorld2.jar")); //Wait for the monitor to notice Thread.sleep(2000); @@ -129,7 +122,7 @@ public class AccumuloReloadingVFSClassLo @After public void tearDown() throws Exception { - this.hdfs.delete(TEST_DIR, true); + folder1.delete(); } } Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java?rev=1431579&r1=1431578&r2=1431579&view=diff ============================================================================== --- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java (original) +++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java Thu Jan 10 19:47:20 2013 @@ -17,16 +17,18 @@ package org.apache.accumulo.start.classloader.vfs; import java.io.File; -import java.net.URL; +import java.io.FileWriter; +import java.io.IOException; import java.net.URLClassLoader; import org.apache.accumulo.start.classloader.AccumuloClassLoader; -import org.apache.accumulo.test.AccumuloDFSBase; +import org.apache.commons.io.FileUtils; import org.apache.commons.vfs2.impl.VFSClassLoader; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -39,9 +41,20 @@ import org.powermock.reflect.Whitebox; @SuppressStaticInitializationFor({"org.apache.accumulo.start.classloader.AccumuloVFSClassLoader", "org.apache.log4j.LogManager"}) @PowerMockIgnore({"org.apache.log4j.*", "org.apache.hadoop.log.metrics", "org.apache.commons.logging.*", "org.xml.*", "javax.xml.*", "org.w3c.dom.*", "org.apache.hadoop.*"}) -public class AccumuloVFSClassLoaderTest extends AccumuloDFSBase { +public class AccumuloVFSClassLoaderTest { + private TemporaryFolder folder1 = new TemporaryFolder(); + + @Before + public void setup() throws IOException { + folder1.create(); + } + @After + public void tearDown() { + folder1.delete(); + } + /* * Test that if enabled, but not configured, that the code creates the 2nd level classloader */ @@ -50,17 +63,26 @@ public class AccumuloVFSClassLoaderTest Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader) null); - URL defaultDir = this.getClass().getResource("/disabled"); - - Whitebox.setInternalState(AccumuloClassLoader.class, "SITE_CONF", new File(defaultDir.getPath() + "/conf/accumulo-site.xml").toURI().toURL().toString()); + File conf = folder1.newFile("accumulo-site.xml"); + FileWriter out = new FileWriter(conf); + out.append("\n"); + out.append("\n"); + out.append("\n"); + out.append("general.classpaths\n"); + out.append("\n"); + out.append("\n"); + out.append("\n"); + out.append("general.vfs.classpaths\n"); + out.append("\n"); + out.append("\n"); + out.append("\n"); + out.close(); + + Whitebox.setInternalState(AccumuloClassLoader.class, "SITE_CONF", conf.toURI().toURL().toString()); Whitebox.setInternalState(AccumuloVFSClassLoader.class, "lock", new Object()); ClassLoader acl = AccumuloVFSClassLoader.getClassLoader(); Assert.assertTrue((acl instanceof VFSClassLoader)); Assert.assertTrue((acl.getParent() instanceof URLClassLoader)); - - // URLClassLoader ucl = (URLClassLoader) acl; - // URL[] classpath = ucl.getURLs(); - // System.out.println(Arrays.toString(classpath)); } /* @@ -71,21 +93,26 @@ public class AccumuloVFSClassLoaderTest Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader) null); - - // Create default context directory - FileSystem hdfs = cluster.getFileSystem(); - Path DEFAULT = new Path("/accumulo/classpath"); - hdfs.mkdirs(DEFAULT); // Copy jar file to TEST_DIR - URL jarPath = this.getClass().getResource("/HelloWorld.jar"); - Path src = new Path(jarPath.toURI().toString()); - Path dst = new Path(DEFAULT, src.getName()); - hdfs.copyFromLocalFile(src, dst); - - URL defaultDir = this.getClass().getResource("/default"); + FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"), folder1.newFile("HelloWorld.jar")); - Whitebox.setInternalState(AccumuloClassLoader.class, "SITE_CONF", new File(defaultDir.getPath() + "/conf/accumulo-site.xml").toURI().toURL().toString()); + File conf = folder1.newFile("accumulo-site.xml"); + FileWriter out = new FileWriter(conf); + out.append("\n"); + out.append("\n"); + out.append("\n"); + out.append("general.classpaths\n"); + out.append("\n"); + out.append("\n"); + out.append("\n"); + out.append("general.vfs.classpaths\n"); + out.append("" + new File(folder1.getRoot(), "HelloWorld.jar").toURI() + "\n"); + out.append("\n"); + out.append("\n"); + out.close(); + + Whitebox.setInternalState(AccumuloClassLoader.class, "SITE_CONF", conf.toURI().toURL().toString()); Whitebox.setInternalState(AccumuloVFSClassLoader.class, "lock", new Object()); ClassLoader acl = AccumuloVFSClassLoader.getClassLoader(); Assert.assertTrue((acl instanceof VFSClassLoader)); @@ -93,14 +120,11 @@ public class AccumuloVFSClassLoaderTest VFSClassLoader arvcl = (VFSClassLoader) acl.getParent(); Assert.assertEquals(1, arvcl.getFileObjects().length); // We can't be sure what the authority/host will be due to FQDN mappings, so just check the path - Assert.assertTrue(arvcl.getFileObjects()[0].getURL().toString().contains("/accumulo/classpath/HelloWorld.jar")); + Assert.assertTrue(arvcl.getFileObjects()[0].getURL().toString().contains("HelloWorld.jar")); Class clazz1 = arvcl.loadClass("test.HelloWorld"); Object o1 = clazz1.newInstance(); Assert.assertEquals("Hello World!", o1.toString()); Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader) null); - - hdfs.delete(DEFAULT, true); - } } Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java?rev=1431579&r1=1431578&r2=1431579&view=diff ============================================================================== --- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java (original) +++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java Thu Jan 10 19:47:20 2013 @@ -16,43 +16,97 @@ */ package org.apache.accumulo.start.classloader.vfs; -import java.net.URL; +import java.io.File; +import java.net.URI; import java.util.HashSet; import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig; import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextsConfig; -import org.apache.accumulo.test.AccumuloDFSBase; +import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider; +import org.apache.commons.io.FileUtils; +import org.apache.commons.vfs2.CacheStrategy; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.cache.DefaultFilesCache; +import org.apache.commons.vfs2.cache.SoftRefFilesCache; +import org.apache.commons.vfs2.impl.DefaultFileReplicator; +import org.apache.commons.vfs2.impl.DefaultFileSystemManager; +import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; import org.apache.commons.vfs2.impl.VFSClassLoader; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.rules.TemporaryFolder; -public class ContextManagerTest extends AccumuloDFSBase { +public class ContextManagerTest { - private static final Path TEST_DIR = new Path(HDFS_URI + "/test-dir"); - private static final Path TEST_DIR2 = new Path(HDFS_URI + "/test-dir2"); - - private FileSystem hdfs = null; + private TemporaryFolder folder1 = new TemporaryFolder(); + private TemporaryFolder folder2 = new TemporaryFolder(); + private DefaultFileSystemManager vfs; + private URI uri1; + private URI uri2; + + static DefaultFileSystemManager getVFS() { + DefaultFileSystemManager vfs = new DefaultFileSystemManager(); + try { + vfs.setFilesCache(new DefaultFilesCache()); + vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider()); + vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider()); + vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider()); + vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider()); + vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider()); + vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider()); + vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider()); + vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider()); + vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider()); + vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider()); + vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider()); + vfs.addProvider("hdfs", new HdfsFileProvider()); + vfs.addExtensionMap("jar", "jar"); + vfs.addExtensionMap("zip", "zip"); + vfs.addExtensionMap("gz", "gz"); + vfs.addExtensionMap("tar", "tar"); + vfs.addExtensionMap("tbz2", "tar"); + vfs.addExtensionMap("tgz", "tar"); + vfs.addExtensionMap("bz2", "bz2"); + vfs.addMimeTypeMap("application/x-tar", "tar"); + vfs.addMimeTypeMap("application/x-gzip", "gz"); + vfs.addMimeTypeMap("application/zip", "zip"); + vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory()); + vfs.setFilesCache(new SoftRefFilesCache()); + vfs.setReplicator(new DefaultFileReplicator()); + vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE); + vfs.init(); + } catch (FileSystemException e) { + throw new RuntimeException("Error setting up VFS", e); + } + + return vfs; + } @Before public void setup() throws Exception { - this.hdfs = cluster.getFileSystem(); - this.hdfs.mkdirs(TEST_DIR); - this.hdfs.mkdirs(TEST_DIR2); - - //Copy jar file to TEST_DIR - URL jarPath = this.getClass().getResource("/HelloWorld.jar"); - Path src = new Path(jarPath.toURI().toString()); - Path dst = new Path(TEST_DIR, src.getName()); - this.hdfs.copyFromLocalFile(src, dst); + + vfs = getVFS(); - Path dst2 = new Path(TEST_DIR2, src.getName()); - this.hdfs.copyFromLocalFile(src, dst2); + folder1.create(); + folder2.create(); + + FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"), folder1.newFile("HelloWorld.jar")); + FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"), folder2.newFile("HelloWorld.jar")); + + uri1 = new File(folder1.getRoot(), "HelloWorld.jar").toURI(); + uri2 = folder2.getRoot().toURI(); } @@ -82,21 +136,21 @@ public class ContextManagerTest extends @Override public ContextConfig getContextConfig(String context) { if (context.equals("CX1")) { - return new ContextConfig(new Path(TEST_DIR, "HelloWorld.jar").toUri().toString(), true); + return new ContextConfig(uri1.toString(), true); } else if (context.equals("CX2")) { - return new ContextConfig(new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString(), true); + return new ContextConfig(uri2.toString(), true); } return null; } }); - FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString()); + FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString()); FileObject[] dirContents = testDir.getChildren(); ClassLoader cl1 = cm.getClassLoader("CX1"); FileObject[] files = ((VFSClassLoader) cl1).getFileObjects(); Assert.assertArrayEquals(createFileSystems(dirContents), files); - FileObject testDir2 = vfs.resolveFile(TEST_DIR2.toUri().toString()); + FileObject testDir2 = vfs.resolveFile(folder2.getRoot().toURI().toString()); FileObject[] dirContents2 = testDir2.getChildren(); ClassLoader cl2 = cm.getClassLoader("CX2"); FileObject[] files2 = ((VFSClassLoader) cl2).getFileObjects(); @@ -117,7 +171,7 @@ public class ContextManagerTest extends @Test public void testPostDelegation() throws Exception { - final VFSClassLoader parent = new VFSClassLoader(new FileObject[] {vfs.resolveFile(new Path(TEST_DIR, "HelloWorld.jar").toUri().toString())}, vfs); + final VFSClassLoader parent = new VFSClassLoader(new FileObject[] {vfs.resolveFile(uri1.toString())}, vfs); Class pclass = parent.loadClass("test.HelloWorld"); @@ -132,9 +186,9 @@ public class ContextManagerTest extends @Override public ContextConfig getContextConfig(String context) { if (context.equals("CX1")) { - return new ContextConfig(new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString(), true); + return new ContextConfig(uri2.toString(), true); } else if (context.equals("CX2")) { - return new ContextConfig(new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString(), false); + return new ContextConfig(uri2.toString(), false); } return null; } @@ -146,7 +200,8 @@ public class ContextManagerTest extends @After public void tearDown() throws Exception { - this.hdfs.delete(TEST_DIR, true); + folder1.delete(); + folder2.delete(); } }