Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6EE7FDAFC for ; Thu, 22 Nov 2012 00:55:57 +0000 (UTC) Received: (qmail 62662 invoked by uid 500); 22 Nov 2012 00:55:57 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 62631 invoked by uid 500); 22 Nov 2012 00:55:57 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 62624 invoked by uid 99); 22 Nov 2012 00:55:57 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Nov 2012 00:55:57 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Nov 2012 00:55:52 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8FFBF2388993 for ; Thu, 22 Nov 2012 00:55:32 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1412398 [1/2] - in /accumulo/trunk: ./ bin/ conf/examples/vfs-classloader/ core/src/main/java/org/apache/accumulo/core/client/mock/ core/src/main/java/org/apache/accumulo/core/conf/ core/src/main/java/org/apache/accumulo/core/file/ core/sr... Date: Thu, 22 Nov 2012 00:55:28 -0000 To: commits@accumulo.apache.org From: dlmarion@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121122005532.8FFBF2388993@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: dlmarion Date: Thu Nov 22 00:55:22 2012 New Revision: 1412398 URL: http://svn.apache.org/viewvc?rev=1412398&view=rev Log: ACCUMULO-708 initial implementation of VFS class loader Added: accumulo/trunk/conf/examples/vfs-classloader/ accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml accumulo/trunk/core/src/test/resources/disabled/ accumulo/trunk/core/src/test/resources/disabled/conf/ accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java accumulo/trunk/start/src/main/java/org/apache/commons/ accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/ accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/ accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileSystemConfigBuilder.java accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsReadOnlyFileContentInfoFactory.java accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsReadOnlyRandomAccessContent.java accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/ReadOnlyHdfsFileProvider.java accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/ReadOnlyHdfsFileSystem.java accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java accumulo/trunk/start/src/test/java/org/apache/accumulo/test/ accumulo/trunk/start/src/test/java/org/apache/accumulo/test/AccumuloDFSBase.java accumulo/trunk/start/src/test/java/org/apache/commons/ accumulo/trunk/start/src/test/java/org/apache/commons/vfs2/ accumulo/trunk/start/src/test/java/org/apache/commons/vfs2/provider/ accumulo/trunk/start/src/test/java/org/apache/commons/vfs2/provider/ReadOnlyHdfsFileProviderTest.java accumulo/trunk/start/src/test/java/org/apache/commons/vfs2/provider/VfsClassLoaderTest.java accumulo/trunk/start/src/test/resources/HelloWorld.jar (with props) accumulo/trunk/start/src/test/resources/application1/ accumulo/trunk/start/src/test/resources/application1/conf/ accumulo/trunk/start/src/test/resources/application1/conf/accumulo-site.xml accumulo/trunk/start/src/test/resources/default/ accumulo/trunk/start/src/test/resources/default/conf/ accumulo/trunk/start/src/test/resources/default/conf/accumulo-site.xml accumulo/trunk/start/src/test/resources/disabled/ accumulo/trunk/start/src/test/resources/disabled/conf/ accumulo/trunk/start/src/test/resources/disabled/conf/accumulo-site.xml accumulo/trunk/test/src/test/resources/ accumulo/trunk/test/src/test/resources/conf/ accumulo/trunk/test/src/test/resources/conf/accumulo-site.xml accumulo/trunk/test/src/test/resources/lib/ accumulo/trunk/test/src/test/resources/lib/ext/ Modified: accumulo/trunk/bin/accumulo accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java accumulo/trunk/pom.xml accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java accumulo/trunk/start/pom.xml accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java Modified: accumulo/trunk/bin/accumulo URL: http://svn.apache.org/viewvc/accumulo/trunk/bin/accumulo?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/bin/accumulo (original) +++ accumulo/trunk/bin/accumulo Thu Nov 22 00:55:22 2012 @@ -24,6 +24,8 @@ bin=`cd "$bin"; pwd` START_JAR=$ACCUMULO_HOME/lib/accumulo-start-$ACCUMULO_VERSION.jar COMMONS_JCI_JARS=$ACCUMULO_HOME/lib/commons-jci-core-1.0.jar:$ACCUMULO_HOME/lib/commons-jci-fam-1.0.jar:$ACCUMULO_HOME/lib/log4j-1.2.16.jar:$ACCUMULO_HOME/lib/commons-logging-1.0.4.jar:$ACCUMULO_HOME/lib/commons-logging-api-1.0.4.jar START_CLASSES_DIR=$ACCUMULO_HOME/src/start/target/classes +COMMONS_VFS_JARS=$ACCUMULO_HOME/lib/commons-vfs2-2.0.jar +HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath` locationByProgram() { @@ -82,7 +84,7 @@ else fi XML_FILES=${ACCUMULO_HOME}/conf -CLASSPATH=${XML_FILES}:${START_JAR}:${COMMONS_JCI_JARS} +CLASSPATH=${XML_FILES}:${START_JAR}:${COMMONS_JCI_JARS}:${COMMONS_VFS_JARS}:${HADOOP_CLASSPATH} if [ -z $JAVA_HOME -o ! -d $JAVA_HOME ]; then echo "JAVA_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh" Added: accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml?rev=1412398&view=auto ============================================================================== --- accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml (added) +++ accumulo/trunk/conf/examples/vfs-classloader/accumulo-site.xml Thu Nov 22 00:55:22 2012 @@ -0,0 +1,135 @@ + + + + + + + + + instance.zookeeper.host + localhost:2181 + comma separated list of zookeeper servers + + + + logger.dir.walog + walogs + The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories. + + + + instance.secret + DEFAULT + A secret unique to a given instance that all servers must know in order to communicate with one another. + Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], + and then update this file. + + + + + tserver.memory.maps.max + 80M + + + + tserver.cache.data.size + 7M + + + + tserver.cache.index.size + 20M + + + + trace.password + + secret + + + + trace.user + root + + + + tserver.sort.buffer.size + 50M + + + + tserver.walog.max.size + 100M + + + + general.classpaths + + $ACCUMULO_HOME/server/target/classes/, + $ACCUMULO_HOME/core/target/classes/, + $ACCUMULO_HOME/start/target/classes/, + $ACCUMULO_HOME/examples/target/classes/, + $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, + $ACCUMULO_HOME/lib/[^.].*.jar, + $ZOOKEEPER_HOME/zookeeper[^.].*.jar, + $HADOOP_HOME/conf, + $HADOOP_HOME/[^.].*.jar, + $HADOOP_HOME/lib/[^.].*.jar, + + Classpaths that accumulo checks for updates and class files. + When using the Security Manager, please remove the ".../target/classes/" values. + + + + + + classloader.vfs.enabled + true + set to true to enable the vfs classloader. If not enabled (default=false), then the old classloader is used + + + + classloader.vfs.context.classpath.system + hdfs://localhost:8020/accumulo/classpath + location of the jars for the default (system) context + + + + classloader.vfs.context.names + application1 + list of context names + + + + classloader.vfs.context.classpath.application1 + hdfs://localhost:8020/application1/classpath + classpath for the application1 context + + + + Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mock/MockInstanceOperations.java Thu Nov 22 00:55:22 2012 @@ -24,7 +24,7 @@ import org.apache.accumulo.core.client.A import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.admin.ActiveScan; import org.apache.accumulo.core.client.admin.InstanceOperations; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; /** * @@ -107,7 +107,7 @@ public class MockInstanceOperations impl @Override public boolean testClassLoad(String className, String asTypeName) throws AccumuloException, AccumuloSecurityException { try { - AccumuloClassLoader.loadClass(className, Class.forName(asTypeName)); + AccumuloVFSClassLoader.loadClass(className, Class.forName(asTypeName)); } catch (ClassNotFoundException e) { e.printStackTrace(); return false; Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java Thu Nov 22 00:55:22 2012 @@ -286,7 +286,18 @@ public enum Property { TABLE_FORMATTER_CLASS("table.formatter", DefaultFormatter.class.getName(), PropertyType.STRING, "The Formatter class to apply on results in the shell"), TABLE_INTERPRETER_CLASS("table.interepreter", DefaultScanInterpreter.class.getName(), PropertyType.STRING, - "The ScanInterpreter class to apply on scan arguments in the shell"); + "The ScanInterpreter class to apply on scan arguments in the shell"), + + + //VFS ClassLoader properties + VFS_CLASSLOADER_PREFIX("classloader.vfs", null, PropertyType.PREFIX, "Properties in this category affect the VFS ClassLoader"), + VFS_CLASSLOADER_ENABLED("classloader.vfs.enabled", "false", PropertyType.BOOLEAN, "Enable/disable VFS Classloader"), + VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY("classloader.vfs.context.classpath.system", "", PropertyType.STRING, + "Classpath for the system context"), + VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY("classloader.vfs.context.names", "", PropertyType.STRING, + "Comma separated list of context names"), + VFS_CONTEXT_CLASSPATH_PROPERTY("classloader.vfs.context.classpath.", null, PropertyType.PREFIX, "Classpath for this context"); + private String key, defaultValue, description; private PropertyType type; Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java Thu Nov 22 00:55:22 2012 @@ -51,7 +51,7 @@ import org.apache.accumulo.core.iterator import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.LoggingRunnable; import org.apache.accumulo.core.util.NamingThreadFactory; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.Text; @@ -113,7 +113,7 @@ public class BloomFilterLayer { * load KeyFunctor */ try { - Class clazz = AccumuloClassLoader.loadClass(acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR), KeyFunctor.class); + Class clazz = AccumuloVFSClassLoader.loadClass(acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR), KeyFunctor.class); transformer = clazz.newInstance(); } catch (Exception e) { @@ -207,7 +207,7 @@ public class BloomFilterLayer { */ ClassName = in.readUTF(); - Class clazz = AccumuloClassLoader.loadClass(ClassName, KeyFunctor.class); + Class clazz = AccumuloVFSClassLoader.loadClass(ClassName, KeyFunctor.class); transformer = clazz.newInstance(); /** Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java Thu Nov 22 00:55:22 2012 @@ -29,7 +29,7 @@ import org.apache.accumulo.core.data.Ran import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.aggregation.Aggregator; import org.apache.accumulo.core.iterators.conf.ColumnToClassMapping; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.log4j.Logger; /** @@ -193,7 +193,7 @@ public class AggregatingIterator impleme return false; Class clazz; try { - clazz = AccumuloClassLoader.loadClass(classname, Aggregator.class); + clazz = AccumuloVFSClassLoader.loadClass(classname, Aggregator.class); clazz.newInstance(); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("class not found: " + classname); Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java Thu Nov 22 00:55:22 2012 @@ -38,7 +38,7 @@ import org.apache.accumulo.core.iterator import org.apache.accumulo.core.iterators.user.VersioningIterator; import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig; import org.apache.accumulo.core.tabletserver.thrift.TIteratorSetting; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.log4j.Logger; @@ -224,7 +224,7 @@ public class IteratorUtil { Class> clazz; if (useAccumuloClassLoader){ - clazz = (Class>) AccumuloClassLoader.loadClass(iterInfo.className, SortedKeyValueIterator.class); + clazz = (Class>) AccumuloVFSClassLoader.loadClass(iterInfo.className, SortedKeyValueIterator.class); }else{ clazz = (Class>) Class.forName(iterInfo.className).asSubclass(SortedKeyValueIterator.class); } Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java Thu Nov 22 00:55:22 2012 @@ -24,7 +24,7 @@ import java.util.NoSuchElementException; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; /** * A Combiner that decodes each Value to type V before reducing, then encodes the result of typedReduce back to Value. @@ -132,7 +132,7 @@ public abstract class TypedValueCombiner protected void setEncoder(String encoderClass) { try { @SuppressWarnings("unchecked") - Class> clazz = (Class>) AccumuloClassLoader.loadClass(encoderClass, Encoder.class); + Class> clazz = (Class>) AccumuloVFSClassLoader.loadClass(encoderClass, Encoder.class); encoder = clazz.newInstance(); } catch (ClassNotFoundException e) { throw new IllegalArgumentException(e); Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnToClassMapping.java Thu Nov 22 00:55:22 2012 @@ -24,7 +24,7 @@ import org.apache.accumulo.core.data.Key import org.apache.accumulo.core.iterators.conf.ColumnUtil.ColFamHashKey; import org.apache.accumulo.core.iterators.conf.ColumnUtil.ColHashKey; import org.apache.accumulo.core.util.Pair; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.hadoop.io.Text; public class ColumnToClassMapping { @@ -50,7 +50,7 @@ public class ColumnToClassMapping { Pair pcic = ColumnSet.decodeColumns(column); - Class clazz = AccumuloClassLoader.loadClass(className, c); + Class clazz = AccumuloVFSClassLoader.loadClass(className, c); if (pcic.getSecond() == null) { addObject(pcic.getFirst(), clazz.newInstance()); Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ClasspathCommand.java Thu Nov 22 00:55:22 2012 @@ -18,13 +18,13 @@ package org.apache.accumulo.core.util.sh import org.apache.accumulo.core.util.shell.Shell; import org.apache.accumulo.core.util.shell.Shell.Command; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.commons.cli.CommandLine; public class ClasspathCommand extends Command { @Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) { - AccumuloClassLoader.printClassPath(); + AccumuloVFSClassLoader.printClassPath(); return 0; } Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java Thu Nov 22 00:55:22 2012 @@ -39,7 +39,7 @@ import org.apache.accumulo.core.util.int import org.apache.accumulo.core.util.shell.Shell; import org.apache.accumulo.core.util.shell.Shell.Command; import org.apache.accumulo.core.util.shell.Shell.PrintFile; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -166,9 +166,9 @@ public class ScanCommand extends Command Class clazz = null; try { if (cl.hasOption(interpreterOpt.getOpt())) { - clazz = AccumuloClassLoader.loadClass(cl.getOptionValue(interpreterOpt.getOpt()), ScanInterpreter.class); + clazz = AccumuloVFSClassLoader.loadClass(cl.getOptionValue(interpreterOpt.getOpt()), ScanInterpreter.class); } else if (cl.hasOption(formatterInterpeterOpt.getOpt())) { - clazz = AccumuloClassLoader.loadClass(cl.getOptionValue(formatterInterpeterOpt.getOpt()), ScanInterpreter.class); + clazz = AccumuloVFSClassLoader.loadClass(cl.getOptionValue(formatterInterpeterOpt.getOpt()), ScanInterpreter.class); } } catch (ClassNotFoundException e) { shellState.getReader().printString("Interpreter class could not be loaded.\n" + e.getMessage() + "\n"); @@ -187,10 +187,10 @@ public class ScanCommand extends Command try { if (cl.hasOption(formatterOpt.getOpt())) { - return AccumuloClassLoader.loadClass(cl.getOptionValue(formatterOpt.getOpt()), Formatter.class); + return AccumuloVFSClassLoader.loadClass(cl.getOptionValue(formatterOpt.getOpt()), Formatter.class); } else if (cl.hasOption(formatterInterpeterOpt.getOpt())) { - return AccumuloClassLoader.loadClass(cl.getOptionValue(formatterInterpeterOpt.getOpt()), Formatter.class); + return AccumuloVFSClassLoader.loadClass(cl.getOptionValue(formatterInterpeterOpt.getOpt()), Formatter.class); } } catch (ClassNotFoundException e) { shellState.getReader().printString("Formatter class could not be loaded.\n" + e.getMessage() + "\n"); Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java Thu Nov 22 00:55:22 2012 @@ -43,7 +43,7 @@ import org.apache.accumulo.core.util.she import org.apache.accumulo.core.util.shell.Shell.Command; import org.apache.accumulo.core.util.shell.ShellCommandException; import org.apache.accumulo.core.util.shell.ShellCommandException.ErrorCode; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionGroup; @@ -123,7 +123,7 @@ public class SetIterCommand extends Comm OptionDescriber skvi; Class clazz; try { - clazz = AccumuloClassLoader.loadClass(className, OptionDescriber.class); + clazz = AccumuloVFSClassLoader.loadClass(className, OptionDescriber.class); skvi = clazz.newInstance(); } catch (ClassNotFoundException e) { throw new IllegalArgumentException(e.getMessage()); Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java (original) +++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ShellPluginConfigurationCommand.java Thu Nov 22 00:55:22 2012 @@ -25,7 +25,7 @@ import org.apache.accumulo.core.client.T import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.util.shell.Shell; import org.apache.accumulo.core.util.shell.Shell.Command; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionGroup; @@ -102,7 +102,7 @@ public abstract class ShellPluginConfigu if (ent.getKey().equals(pluginProp.toString())) { Class pluginClazz; try { - pluginClazz = AccumuloClassLoader.loadClass(ent.getValue(), clazz); + pluginClazz = AccumuloVFSClassLoader.loadClass(ent.getValue(), clazz); } catch (ClassNotFoundException e) { Logger.getLogger(ShellPluginConfigurationCommand.class).warn("Class not found" + e.getMessage()); return null; Added: accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml?rev=1412398&view=auto ============================================================================== --- accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml (added) +++ accumulo/trunk/core/src/test/resources/disabled/conf/accumulo-site.xml Thu Nov 22 00:55:22 2012 @@ -0,0 +1,108 @@ + + + + + + + + + instance.zookeeper.host + localhost:2181 + comma separated list of zookeeper servers + + + + logger.dir.walog + walogs + The directory used to store write-ahead logs on the local + filesystem. It is possible to specify a comma-separated list of + directories. + + + + instance.secret + DEFAULT + A secret unique to a given instance that all servers must + know in order to communicate with one another. + Change it before initialization. To change it later use ./bin/accumulo + org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], + and then update this file. + + + + + tserver.memory.maps.max + 80M + + + + tserver.cache.data.size + 7M + + + + tserver.cache.index.size + 20M + + + + trace.password + + secret + + + + trace.user + root + + + + tserver.sort.buffer.size + 50M + + + + tserver.walog.max.size + 100M + + + + general.classpaths + + $ACCUMULO_HOME/server/target/classes/, + $ACCUMULO_HOME/core/target/classes/, + $ACCUMULO_HOME/start/target/classes/, + $ACCUMULO_HOME/examples/target/classes/, + $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, + $ACCUMULO_HOME/lib/[^.].*.jar, + $ZOOKEEPER_HOME/zookeeper[^.].*.jar, + $HADOOP_HOME/conf, + $HADOOP_HOME/[^.].*.jar, + $HADOOP_HOME/lib/[^.].*.jar, + + Classpaths that accumulo checks for updates and class + files. + When using the Security Manager, please remove the ".../target/classes/" + values. + + + + + + vfs.classloader.enabled + true + + + Modified: accumulo/trunk/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/pom.xml?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/pom.xml (original) +++ accumulo/trunk/pom.xml Thu Nov 22 00:55:22 2012 @@ -144,7 +144,7 @@ ../lib - commons-collections,commons-configuration,commons-io,commons-lang,jline,log4j,libthrift,commons-jci-core,commons-jci-fam,commons-logging,commons-logging-api + commons-collections,commons-configuration,commons-io,commons-lang,jline,log4j,libthrift,commons-jci-core,commons-jci-fam,commons-logging,commons-logging-api,commons-vfs2 true @@ -197,7 +197,7 @@ maven-surefire-plugin - .. + ../test/src/test/resources Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java Thu Nov 22 00:55:22 2012 @@ -47,7 +47,7 @@ import org.apache.accumulo.server.conf.S import org.apache.accumulo.server.security.Authenticator; import org.apache.accumulo.server.security.ZKAuthenticator; import org.apache.accumulo.server.zookeeper.TransactionWatcher; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.log4j.Logger; import org.apache.thrift.TException; @@ -279,7 +279,7 @@ public class ClientServiceHandler implem Class shouldMatch; try { shouldMatch = loader.loadClass(interfaceMatch); - Class test = AccumuloClassLoader.loadClass(className, shouldMatch); + Class test = AccumuloVFSClassLoader.loadClass(className, shouldMatch); test.newInstance(); return true; } catch (ClassCastException e) { Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/ConstraintLoader.java Thu Nov 22 00:55:22 2012 @@ -22,7 +22,7 @@ import java.util.Map.Entry; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.constraints.Constraint; import org.apache.accumulo.server.conf.TableConfiguration; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.log4j.Logger; public class ConstraintLoader { @@ -35,7 +35,7 @@ public class ConstraintLoader { for (Entry entry : conf) { if (entry.getKey().startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey())) { String className = entry.getValue(); - Class clazz = AccumuloClassLoader.loadClass(className, Constraint.class); + Class clazz = AccumuloVFSClassLoader.loadClass(className, Constraint.class); log.debug("Loaded constraint " + clazz.getName() + " for " + conf.getTableId()); cc.addConstraint(clazz.newInstance()); } Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java Thu Nov 22 00:55:22 2012 @@ -157,7 +157,7 @@ import org.apache.accumulo.server.util.T import org.apache.accumulo.server.util.time.SimpleTimer; import org.apache.accumulo.server.zookeeper.ZooLock; import org.apache.accumulo.server.zookeeper.ZooReaderWriter; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -513,7 +513,7 @@ public class Master implements LiveTServ T instance = null; try { - Class clazz = AccumuloClassLoader.loadClass(clazzName, base); + Class clazz = AccumuloVFSClassLoader.loadClass(clazzName, base); instance = clazz.newInstance(); log.info("Loaded class : " + clazzName); } catch (Exception e) { Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java Thu Nov 22 00:55:22 2012 @@ -35,7 +35,7 @@ import org.apache.accumulo.server.client import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.master.state.TabletMigration; import org.apache.accumulo.server.security.SecurityConstants; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.log4j.Logger; public class TableLoadBalancer extends TabletBalancer { @@ -45,7 +45,7 @@ public class TableLoadBalancer extends T Map perTableBalancers = new HashMap(); private TabletBalancer constructNewBalancerForTable(String clazzName, String table) throws Exception { - Class clazz = AccumuloClassLoader.loadClass(clazzName, TabletBalancer.class); + Class clazz = AccumuloVFSClassLoader.loadClass(clazzName, TabletBalancer.class); Constructor constructor = clazz.getConstructor(String.class); return constructor.newInstance(table); } Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServerResourceManager.java Thu Nov 22 00:55:22 2012 @@ -45,14 +45,14 @@ import org.apache.accumulo.core.data.Key import org.apache.accumulo.core.file.blockfile.cache.LruBlockCache; import org.apache.accumulo.core.util.Daemon; import org.apache.accumulo.core.util.LoggingRunnable; -import org.apache.accumulo.core.util.NamingThreadFactory; import org.apache.accumulo.core.util.MetadataTable.DataFileValue; +import org.apache.accumulo.core.util.NamingThreadFactory; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.tabletserver.FileManager.ScanFileManager; import org.apache.accumulo.server.tabletserver.Tablet.MajorCompactionReason; import org.apache.accumulo.server.util.time.SimpleTimer; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.Logger; @@ -199,7 +199,7 @@ public class TabletServerResourceManager fileManager = new FileManager(conf, fs, maxOpenFiles, _dCache, _iCache); try { - Class clazz = AccumuloClassLoader.loadClass(acuConf.get(Property.TSERV_MEM_MGMT), MemoryManager.class); + Class clazz = AccumuloVFSClassLoader.loadClass(acuConf.get(Property.TSERV_MEM_MGMT), MemoryManager.class); memoryManager = clazz.newInstance(); memoryManager.init(conf); log.debug("Loaded memory manager : " + memoryManager.getClass().getName()); Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/functional/FunctionalTest.java Thu Nov 22 00:55:22 2012 @@ -42,7 +42,7 @@ import org.apache.accumulo.core.data.Ran import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.thrift.AuthInfo; import org.apache.accumulo.server.conf.ServerConfiguration; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; @@ -290,7 +290,7 @@ public abstract class FunctionalTest { String clazz = remainingArgs[0]; String opt = remainingArgs[1]; - Class testClass = AccumuloClassLoader.loadClass(clazz, FunctionalTest.class); + Class testClass = AccumuloVFSClassLoader.loadClass(clazz, FunctionalTest.class); FunctionalTest fTest = testClass.newInstance(); fTest.setMaster(master); Modified: accumulo/trunk/start/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/start/pom.xml?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/start/pom.xml (original) +++ accumulo/trunk/start/pom.xml Thu Nov 22 00:55:22 2012 @@ -46,6 +46,13 @@ + + maven-surefire-plugin + + always + + + @@ -67,6 +74,53 @@ commons-io commons-io + + org.apache.commons + commons-vfs2 + 2.0 + + + + org.apache.hadoop + hadoop-test + 0.20.205.0 + test + + + junit + junit + test + + + javax.ws.rs + jsr311-api + 1.0 + test + + + org.slf4j + slf4j-nop + 1.7.2 + test + + + org.powermock + powermock-module-junit4 + 1.4.12 + test + + + org.powermock + powermock-api-easymock + 1.4.12 + test + + + org.easymock + easymock + 3.1 + test + Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java?rev=1412398&r1=1412397&r2=1412398&view=diff ============================================================================== --- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java (original) +++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java Thu Nov 22 00:55:22 2012 @@ -19,7 +19,7 @@ package org.apache.accumulo.start; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; public class Main { @@ -36,36 +36,37 @@ public class Main { Class runTMP = null; - Thread.currentThread().setContextClassLoader(AccumuloClassLoader.getClassLoader()); + ClassLoader cl = AccumuloVFSClassLoader.getClassLoader(); + Thread.currentThread().setContextClassLoader(cl); if (args[0].equals("master")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.master.Master"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.master.Master"); } else if (args[0].equals("tserver")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.tabletserver.TabletServer"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.tabletserver.TabletServer"); } else if (args[0].equals("shell")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.util.shell.Shell"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.core.util.shell.Shell"); } else if (args[0].equals("init")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.util.Initialize"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.util.Initialize"); } else if (args[0].equals("admin")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.util.Admin"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.util.Admin"); } else if (args[0].equals("gc")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.gc.SimpleGarbageCollector"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.gc.SimpleGarbageCollector"); } else if (args[0].equals("monitor")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.monitor.Monitor"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.monitor.Monitor"); } else if (args[0].equals("tracer")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.server.trace.TraceServer"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.server.trace.TraceServer"); } else if (args[0].equals("classpath")) { - AccumuloClassLoader.printClassPath(); + AccumuloVFSClassLoader.printClassPath(); return; } else if (args[0].equals("version")) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.Constants"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.core.Constants"); System.out.println(runTMP.getField("VERSION").get(null)); return; } else if (args[0].equals("rfile-info") ) { - runTMP = AccumuloClassLoader.loadClass("org.apache.accumulo.core.file.rfile.PrintInfo"); + runTMP = AccumuloVFSClassLoader.loadClass("org.apache.accumulo.core.file.rfile.PrintInfo"); } else { try { - runTMP = AccumuloClassLoader.loadClass(args[0]); + runTMP = AccumuloVFSClassLoader.loadClass(args[0]); } catch (ClassNotFoundException cnfe) { System.out.println("Classname " + args[0] + " not found. Please make sure you use the wholly qualified package name."); System.exit(1); @@ -96,7 +97,8 @@ public class Main { }; Thread t = new Thread(r, args[0]); - t.setContextClassLoader(AccumuloClassLoader.getClassLoader()); + ClassLoader tcl = AccumuloVFSClassLoader.getClassLoader(); + t.setContextClassLoader(tcl); t.start(); } catch (Throwable t) { System.err.println("Uncaught exception: " + t.getMessage()); Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java?rev=1412398&view=auto ============================================================================== --- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java (added) +++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java Thu Nov 22 00:55:22 2012 @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.start.classloader.vfs; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.SecureClassLoader; +import java.util.Enumeration; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemManager; +import org.apache.log4j.Logger; + +/** + * Classloader that delegates requests to a classloader mapped to a specific context + * + * + */ +public class AccumuloContextClassLoader extends SecureClassLoader { + + private static final Logger log = Logger.getLogger(AccumuloContextClassLoader.class); + + public static final String DEFAULT_CONTEXT = "SYSTEM"; + + private Map loaders = new ConcurrentHashMap(); + private FileSystemManager vfs = null; + private ClassLoader parent = null; + + public AccumuloContextClassLoader(FileObject[] defaultClassPath, FileSystemManager vfs, ClassLoader parent) throws FileSystemException { + this.vfs = vfs; + this.parent = parent; + loaders.put(DEFAULT_CONTEXT, new AccumuloReloadingVFSClassLoader(defaultClassPath, vfs, parent)); + } + + public synchronized void addContext(String context, FileObject[] files) throws FileSystemException { + if (!loaders.containsKey(context)) + loaders.put(context, new AccumuloReloadingVFSClassLoader(files, vfs, parent)); + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + return loadClass(DEFAULT_CONTEXT, name); + } + + public Class loadClass(String context, String name) throws ClassNotFoundException { + if (null == context) + context = DEFAULT_CONTEXT; + return loaders.get(context).loadClass(name); + } + + @Override + public URL getResource(String name) { + return getResource(DEFAULT_CONTEXT, name); + } + + public URL getResource(String context, String name) { + if (null == context) + context = DEFAULT_CONTEXT; + return loaders.get(context).getResource(name); + } + + @Override + public Enumeration getResources(String name) throws IOException { + return getResources(DEFAULT_CONTEXT, name); + } + + public Enumeration getResources(String context, String name) throws IOException { + if (null == context) + context = DEFAULT_CONTEXT; + return loaders.get(context).getResources(name); + } + + @Override + public InputStream getResourceAsStream(String name) { + return getResourceAsStream(DEFAULT_CONTEXT, name); + } + + public InputStream getResourceAsStream(String context, String name) { + if (null == context) + context = DEFAULT_CONTEXT; + return loaders.get(context).getResourceAsStream(name); + } + + @Override + public synchronized void setDefaultAssertionStatus(boolean enabled) { + setDefaultAssertionStatus(DEFAULT_CONTEXT, enabled); + } + + public synchronized void setDefaultAssertionStatus(String context, boolean enabled) { + if (null == context) + context = DEFAULT_CONTEXT; + loaders.get(context).setDefaultAssertionStatus(enabled); + } + + @Override + public synchronized void setPackageAssertionStatus(String packageName, boolean enabled) { + setPackageAssertionStatus(DEFAULT_CONTEXT, packageName, enabled); + } + + public synchronized void setPackageAssertionStatus(String context, String packageName, boolean enabled) { + if (null == context) + context = DEFAULT_CONTEXT; + loaders.get(context).setPackageAssertionStatus(packageName, enabled); + } + + @Override + public synchronized void setClassAssertionStatus(String className, boolean enabled) { + setClassAssertionStatus(DEFAULT_CONTEXT, className, enabled); + } + + public synchronized void setClassAssertionStatus(String context, String className, boolean enabled) { + if (null == context) + context = DEFAULT_CONTEXT; + loaders.get(context).setClassAssertionStatus(className, enabled); + } + + @Override + public synchronized void clearAssertionStatus() { + clearAssertionStatus(DEFAULT_CONTEXT); + } + + public synchronized void clearAssertionStatus(String context) { + if (null == context) + context = DEFAULT_CONTEXT; + loaders.get(context).clearAssertionStatus(); + } + + public AccumuloReloadingVFSClassLoader getClassLoader(String context) { + return loaders.get(context); + } + + public void removeContext(String context) { + if (null != context && !(DEFAULT_CONTEXT.equals(context))) { + AccumuloReloadingVFSClassLoader cl = loaders.remove(context); + cl.close(); + } + } + + public void close() { + for (Entry e : loaders.entrySet()) { + e.getValue().close(); + } + loaders.clear(); + } + + @Override + protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + return this.loadClass(DEFAULT_CONTEXT, name); + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + return findClass(DEFAULT_CONTEXT, name); + } + + protected Class findClass(String context, String name) throws ClassNotFoundException { + if (null == context) + context = DEFAULT_CONTEXT; + return loaders.get(context).findClass(name); + } + + @Override + public String toString() { + URLClassLoader ucl = (URLClassLoader) this.parent; + URL[] uclPaths = ucl.getURLs(); + StringBuilder buf = new StringBuilder(); + buf.append("Parent classpath {\n"); + for (URL u : uclPaths) + buf.append("\t").append(u.toString()).append("\n"); + buf.append("}\nContext classpaths {\n"); + for (Entry entry : this.loaders.entrySet()) { + buf.append("\tContext: ").append(entry.getKey()).append(" {\n"); + for (FileObject f : entry.getValue().getFiles()) { + try { + buf.append("\t\t").append(f.getURL().toString()).append("\n"); + } catch (FileSystemException e) { + log.error("Error getting URL for file", e); + } + } + buf.append("\t}\n"); + } + buf.append("}"); + return buf.toString(); + } + +} Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1412398&view=auto ============================================================================== --- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java (added) +++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java Thu Nov 22 00:55:22 2012 @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.start.classloader.vfs; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.security.SecureClassLoader; +import java.util.Arrays; +import java.util.Enumeration; + +import org.apache.commons.vfs2.FileChangeEvent; +import org.apache.commons.vfs2.FileListener; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemManager; +import org.apache.commons.vfs2.impl.DefaultFileMonitor; +import org.apache.commons.vfs2.impl.VFSClassLoader; +import org.apache.log4j.Logger; + +/** + * Classloader that delegates operations to a VFSClassLoader object. This class also listens + * for changes in any of the files/directories that are in the classpath and will recreate + * the delegate object if there is any change in the classpath. + * + */ +public class AccumuloReloadingVFSClassLoader extends SecureClassLoader implements FileListener { + + private static final Logger log = Logger.getLogger(AccumuloReloadingVFSClassLoader.class); + + /* 5 minute timeout */ + private static final int DEFAULT_TIMEOUT = 300000; + + private FileObject[] files = null; + private FileSystemManager vfs = null; + private ClassLoader parent = null; + private DefaultFileMonitor monitor = null; + private volatile VFSClassLoader cl = null; + + public AccumuloReloadingVFSClassLoader(FileObject[] files, FileSystemManager vfs, ClassLoader parent) throws FileSystemException { + this(files, vfs, parent, DEFAULT_TIMEOUT); + } + + public AccumuloReloadingVFSClassLoader(FileObject[] files, FileSystemManager vfs, ClassLoader parent, long monitorDelay) throws FileSystemException { + this.files = files; + this.vfs = vfs; + this.parent = parent; + + if (null != parent) + cl = new VFSClassLoader(files, vfs, parent); + else + cl = new VFSClassLoader(files, vfs); + + monitor = new DefaultFileMonitor(this); + monitor.setDelay(monitorDelay); + monitor.setRecursive(true); + for (FileObject file : files) + monitor.addFile(file); + monitor.start(); + } + + public FileObject[] getFiles() { + return this.files; + } + + /** + * Should be ok if this is not called because the thread started by DefaultFileMonitor is a daemon thread + */ + public void close() { + monitor.stop(); + } + + public void fileCreated(FileChangeEvent event) throws Exception { + if (log.isDebugEnabled()) + log.debug(event.getFile().getURL().toString() + " created, recreating classloader"); + cl = new VFSClassLoader(files, vfs, parent); + } + + public void fileDeleted(FileChangeEvent event) throws Exception { + if (log.isDebugEnabled()) + log.debug(event.getFile().getURL().toString() + " changed, recreating classloader"); + cl = new VFSClassLoader(files, vfs, parent); + } + + public void fileChanged(FileChangeEvent event) throws Exception { + if (log.isDebugEnabled()) + log.debug(event.getFile().getURL().toString() + " deleted, recreating classloader"); + cl = new VFSClassLoader(files, vfs, parent); + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + return this.cl.loadClass(name); + } + + @Override + protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + return this.loadClass(name); + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + return this.loadClass(name); + } + + @Override + public URL getResource(String name) { + return this.cl.getResource(name); + } + + @Override + public Enumeration getResources(String name) throws IOException { + return this.cl.getResources(name); + } + + @Override + public InputStream getResourceAsStream(String name) { + return this.cl.getResourceAsStream(name); + } + + @Override + public synchronized void setDefaultAssertionStatus(boolean enabled) { + this.cl.setDefaultAssertionStatus(enabled); + } + + @Override + public synchronized void setPackageAssertionStatus(String packageName, boolean enabled) { + this.cl.setPackageAssertionStatus(packageName, enabled); + } + + @Override + public synchronized void setClassAssertionStatus(String className, boolean enabled) { + this.cl.setClassAssertionStatus(className, enabled); + } + + @Override + public synchronized void clearAssertionStatus() { + this.cl.clearAssertionStatus(); + } + + @Override + public String toString() { + return Arrays.toString(this.files); + } + +} Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1412398&view=auto ============================================================================== --- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java (added) +++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java Thu Nov 22 00:55:22 2012 @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.start.classloader.vfs; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.commons.vfs2.CacheStrategy; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.cache.DefaultFilesCache; +import org.apache.commons.vfs2.cache.SoftRefFilesCache; +import org.apache.commons.vfs2.impl.DefaultFileReplicator; +import org.apache.commons.vfs2.impl.DefaultFileSystemManager; +import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory; +import org.apache.commons.vfs2.provider.ReadOnlyHdfsFileProvider; +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; + +/** + * This class builds a hierarchy of Classloaders in the form of: + * + * 
+ * SystemClassLoader that loads JVM classes
+ *       ^
+ *       |
+ * URLClassLoader that references the URLs for HADOOP_HOME, ZOOKEEPER_HOME, ACCUMULO_HOME and their associated directories
+ *       ^
+ *       |
+ * AccumuloContextClassLoader that contains a map of context names to AccumuloReloadingVFSClassLoaders
+ *
+ * + * This class requires new properties in the site configuration file + * + * default.context.classpath -> list of URIs for the default system context. + * classloader.context.names -> name1, name2, name3 + * .context.classpath -> list of URIs for this context + * + */ +public class AccumuloVFSClassLoader { + + public static class AccumuloVFSClassLoaderShutdownThread implements Runnable { + + public void run() { + AccumuloVFSClassLoader.close(); + } + + } + + private static final Logger log = Logger.getLogger(AccumuloVFSClassLoader.class); + + public static final String VFS_CLASSLOADER_ENABLED = "classloader.vfs.enabled"; + + public static final String VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY = "classloader.vfs.context.classpath.system"; + + public static final String VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY = "classloader.vfs.context.names"; + + public static final String VFS_CONTEXT_CLASSPATH_PROPERTY = "classloader.vfs.context.classpath."; + + private static DefaultFileSystemManager vfs = null; + private static ClassLoader parent = null; + private static volatile AccumuloContextClassLoader loader = null; + private static final Object lock = new Object(); + private static final Configuration ACC_CONF = new Configuration(); + private static final String SITE_CONF; + static { + String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml"); + if (System.getenv("ACCUMULO_HOME") != null) { + // accumulo home should be set + SITE_CONF = System.getenv("ACCUMULO_HOME") + "/conf/" + configFile; + } else { + /* + * if not it must be the build-server in which case I use a hack to get unittests working + */ + String userDir = System.getProperty("user.dir"); + if (userDir == null) + throw new RuntimeException("Property user.dir is not set"); + int index = userDir.indexOf("accumulo/"); + if (index >= 0) { + String acuhome = userDir.substring(0, index + "accumulo/".length()); + SITE_CONF = acuhome + "/conf/" + configFile; + } else { + SITE_CONF = "/conf/" + configFile; + } + } + try { + ACC_CONF.addResource(new File(SITE_CONF).toURI().toURL()); + } catch (MalformedURLException e) { + throw new RuntimeException("Unable to create configuration from accumulo-site file: " + SITE_CONF, e); + } + + //Register the shutdown hook + Runtime.getRuntime().addShutdownHook(new Thread(new AccumuloVFSClassLoaderShutdownThread())); + } + + /** + * Replace environment variables in the classpath string with their actual value + * + * @param classpath + * @param env + * @return + */ + private static String replaceEnvVars(String classpath, Map env) { + Pattern envPat = Pattern.compile("\\$[A-Za-z][a-zA-Z0-9_]*"); + Matcher envMatcher = envPat.matcher(classpath); + while (envMatcher.find(0)) { + // name comes after the '$' + String varName = envMatcher.group().substring(1); + String varValue = env.get(varName); + if (varValue == null) { + varValue = ""; + } + classpath = (classpath.substring(0, envMatcher.start()) + varValue + classpath.substring(envMatcher.end())); + envMatcher.reset(classpath); + } + return classpath; + } + + /** + * Populate the list of URLs with the items in the classpath string + * + * @param classpath + * @param urls + * @throws MalformedURLException + */ + private static void addUrl(String classpath, ArrayList urls) throws MalformedURLException { + classpath = classpath.trim(); + if (classpath.length() == 0) + return; + + classpath = replaceEnvVars(classpath, System.getenv()); + + // Try to make a URI out of the classpath + URI uri = null; + try { + uri = new URI(classpath); + } catch (URISyntaxException e) { + // Not a valid URI + } + + if (null == uri || !uri.isAbsolute() || (null != uri.getScheme() && uri.getScheme().equals("file://"))) { + // Then treat this URI as a File. + // This checks to see if the url string is a dir if it expand and get all jars in that directory + final File extDir = new File(classpath); + if (extDir.isDirectory()) + urls.add(extDir.toURI().toURL()); + else { + if (extDir.getParentFile() != null) { + File[] extJars = extDir.getParentFile().listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.matches("^" + extDir.getName()); + } + }); + if (extJars != null && extJars.length > 0) { + for (File jar : extJars) + urls.add(jar.toURI().toURL()); + } + } + } + } else { + urls.add(uri.toURL()); + } + + } + + private static ArrayList findAccumuloURLs() throws IOException { + String cp = ACC_CONF.get(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME, AccumuloClassLoader.ACCUMULO_CLASSPATH_VALUE); + if (cp == null) + return new ArrayList(); + String[] cps = replaceEnvVars(cp, System.getenv()).split(","); + ArrayList urls = new ArrayList(); + for (String classpath : cps) { + if (!classpath.startsWith("#")) { + addUrl(classpath, urls); + } + } + return urls; + } + + private static ClassLoader getAccumuloClassLoader() throws IOException { + ClassLoader parentClassLoader = ClassLoader.getSystemClassLoader(); + ArrayList accumuloURLs = findAccumuloURLs(); + log.debug("Create 2nd tier ClassLoader using URLs: " + accumuloURLs.toString()); + URLClassLoader aClassLoader = new URLClassLoader(accumuloURLs.toArray(new URL[accumuloURLs.size()]), parentClassLoader); + return aClassLoader; + } + + public synchronized static Class loadClass(String classname, Class extension) throws ClassNotFoundException { + try { + return (Class) getClassLoader().loadClass(classname).asSubclass(extension); + } catch (IOException e) { + throw new ClassNotFoundException("IO Error loading class " + classname, e); + } + } + + public static Class loadClass(String classname) throws ClassNotFoundException { + return loadClass(classname, Object.class).asSubclass(Object.class); + } + + public static ClassLoader getClassLoader() throws IOException { + ClassLoader localLoader = loader; + while (null == localLoader) { + synchronized (lock) { + if (null == loader) { + + if (ACC_CONF.getBoolean(VFS_CLASSLOADER_ENABLED, false) == false) { + localLoader = AccumuloClassLoader.getClassLoader(); + return localLoader; + } + + if (null == vfs) { + vfs = new DefaultFileSystemManager(); + //TODO: Might be able to use a different cache impl or specify cache directory in configuration. + vfs.setFilesCache(new DefaultFilesCache()); + vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider()); + vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider()); + vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider()); + vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider()); + vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider()); + vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider()); + vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider()); + vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider()); + vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider()); + vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider()); + vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider()); + vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider()); + vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider()); + vfs.addProvider("hdfs", new ReadOnlyHdfsFileProvider()); + vfs.addExtensionMap("jar", "jar"); + vfs.addExtensionMap("zip", "zip"); + vfs.addExtensionMap("gz", "gz"); + vfs.addExtensionMap("tar", "tar"); + vfs.addExtensionMap("tbz2", "tar"); + vfs.addExtensionMap("tgz", "tar"); + vfs.addExtensionMap("bz2", "bz2"); + vfs.addMimeTypeMap("application/x-tar", "tar"); + vfs.addMimeTypeMap("application/x-gzip", "gz"); + vfs.addMimeTypeMap("application/zip", "zip"); + vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory()); + vfs.setFilesCache(new SoftRefFilesCache()); + vfs.setReplicator(new DefaultFileReplicator()); + vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE); + vfs.init(); + } + + //Set up the 2nd tier class loader + if (null == parent) + parent = getAccumuloClassLoader(); + + //Get the default context classpaths from the configuration + String[] defaultPaths = ACC_CONF.getStrings(VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY); + if (null == defaultPaths || defaultPaths.length == 0) { + log.info("Default context not configured."); + localLoader = parent; + return localLoader; + } + ArrayList defaultClassPath = new ArrayList(); + for (String path : defaultPaths) { + FileObject fo = vfs.resolveFile(path); + if (fo.getType().equals(FileType.FILE)) { + defaultClassPath.add(fo); + } else { + for (FileObject child : fo.getChildren()) { + defaultClassPath.add(child); + } + } + } + + //Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT + loader = new AccumuloContextClassLoader(defaultClassPath.toArray(new FileObject[defaultClassPath.size()]), vfs, parent); + + //Add the other contexts + String[] contexts = ACC_CONF.getStrings(VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY); + if (null != contexts) { + for (String context : contexts) { + String[] contextPaths = ACC_CONF.getStrings(VFS_CONTEXT_CLASSPATH_PROPERTY + context); + if (null != contextPaths) { + ArrayList contextClassPath = new ArrayList(); + for (String cp : contextPaths) { + FileObject fo = vfs.resolveFile(cp); + if (fo.getType().equals(FileType.FILE)) { + contextClassPath.add(fo); + } else { + for (FileObject child : fo.getChildren()) { + contextClassPath.add(child); + } + } + } + log.debug("Creating Context ClassLoader for context: " + context + " using paths: " + contextClassPath.toString()); + loader.addContext(context, contextClassPath.toArray(new FileObject[contextClassPath.size()])); + } + } + } + } + } + localLoader = loader; + } + return localLoader; + } + + public static void printClassPath() { + try { + ClassLoader cl = getClassLoader(); + if (ACC_CONF.getBoolean(VFS_CLASSLOADER_ENABLED, false) == false) { + //If using older classloader, then use its printClassPath method + AccumuloClassLoader.printClassPath(); + } else if (cl instanceof URLClassLoader) { + //If VFS class loader enabled, but no contexts defined. + URLClassLoader ucl = (URLClassLoader) cl; + System.out.println("URL classpath items are: \n"); + for (URL u : ucl.getURLs()) { + System.out.println(u.toExternalForm()); + } + } else if (cl instanceof AccumuloContextClassLoader) { + //If VFS class loader enabled and contexts are defined + System.out.println("VFS classpaths items are:\n" + getClassLoader().toString()); + } else { + System.out.println("Unknown classloader configuration"); + } + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + + + public static void close() { + if (null != vfs) + vfs.close(); + } + +} Added: accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java?rev=1412398&view=auto ============================================================================== --- accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java (added) +++ accumulo/trunk/start/src/main/java/org/apache/commons/vfs2/provider/HdfsFileObject.java Thu Nov 22 00:55:22 2012 @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.vfs2.provider; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.RandomAccessContent; +import org.apache.commons.vfs2.util.RandomAccessMode; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public class HdfsFileObject extends AbstractFileObject { + + public static final String LAST_ACCESS_TIME = "LAST_ACCESS_TIME"; + public static final String BLOCK_SIZE = "BLOCK_SIZE"; + public static final String GROUP = "GROUP"; + public static final String OWNER = "OWNER"; + public static final String PERMISSIONS = "PERMISSIONS"; + public static final String LENGTH = "LENGTH"; + public static final String MODIFICATION_TIME = "MODIFICATION_TIME"; + + private ReadOnlyHdfsFileSystem fs = null; + private FileSystem hdfs = null; + private Path path = null; + private FileStatus stat = null; + + protected HdfsFileObject(AbstractFileName name, ReadOnlyHdfsFileSystem fs, FileSystem hdfs, Path p) { + super(name,fs); + this.fs = fs; + this.hdfs = hdfs; + this.path = p; + } + + @Override + protected void doAttach() throws Exception { + this.stat = this.hdfs.getFileStatus(this.path); + } + + @Override + protected FileType doGetType() throws Exception { + doAttach(); + if (stat.isDir()) + return FileType.FOLDER; + else + return FileType.FILE; + } + + @Override + public boolean exists() throws FileSystemException { + try { + doAttach(); + return this.stat != null; + } catch (FileNotFoundException fne) { + return false; + } catch (Exception e) { + throw new FileSystemException("Unable to check existance ", e); + } + } + + @Override + public boolean canRenameTo(FileObject newfile) { + throw new UnsupportedOperationException(); + } + + @Override + protected String[] doListChildren() throws Exception { + FileStatus[] files = this.hdfs.listStatus(this.path); + String[] children = new String[files.length]; + int i = 0; + for (FileStatus status : files) { + children[i++] = status.getPath().getName(); + } + return children; + } + + @Override + protected long doGetContentSize() throws Exception { + return stat.getLen(); + } + + @Override + protected InputStream doGetInputStream() throws Exception { + return this.hdfs.open(this.path); + } + + @Override + protected boolean doIsHidden() throws Exception { + return false; + } + + @Override + protected boolean doIsReadable() throws Exception { + return true; + } + + @Override + protected boolean doIsWriteable() throws Exception { + return false; + } + + @Override + protected FileObject[] doListChildrenResolved() throws Exception { + String[] children = doListChildren(); + FileObject[] fo = new FileObject[children.length]; + for (int i = 0; i < children.length; i++) { + Path p = new Path(this.path, children[i]); + fo[i] = this.fs.resolveFile(p.toUri().toString()); + } + return fo; + } + + @Override + protected long doGetLastModifiedTime() throws Exception { + if (null != this.stat) + return this.stat.getModificationTime(); + else + return -1; + } + + @Override + protected boolean doSetLastModifiedTime(long modtime) throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + protected Map doGetAttributes() throws Exception { + if (null == this.stat) + return super.doGetAttributes(); + else { + Map attrs = new HashMap(); + attrs.put(LAST_ACCESS_TIME, this.stat.getAccessTime()); + attrs.put(BLOCK_SIZE, this.stat.getBlockSize()); + attrs.put(GROUP, this.stat.getGroup()); + attrs.put(OWNER, this.stat.getOwner()); + attrs.put(PERMISSIONS, this.stat.getPermission().toString()); + attrs.put(LENGTH, this.stat.getLen()); + attrs.put(MODIFICATION_TIME, this.stat.getModificationTime()); + return attrs; + } + } + + @Override + protected void doSetAttribute(String attrName, Object value) throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + protected void doRemoveAttribute(String attrName) throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + protected RandomAccessContent doGetRandomAccessContent(RandomAccessMode mode) throws Exception { + if (mode.equals(RandomAccessMode.READWRITE)) + throw new UnsupportedOperationException(); + return new HdfsReadOnlyRandomAccessContent(this.path, this.hdfs); + } + + @Override + protected boolean doIsSameFile(FileObject destFile) throws FileSystemException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object o) { + if (null == o) return false; + if (o == this) return true; + if (o instanceof HdfsFileObject) { + HdfsFileObject other = (HdfsFileObject) o; + try { + if (other.getURL().equals(this.getURL())) + return true; + } catch (FileSystemException e) { + //TODO: log error + return false; + } + } + return false; + } + + @Override + public int hashCode() { + return this.path.getName().toString().hashCode(); + } + +}