accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1424106 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/ core/src/main/java/org/apache/accumulo/core/conf/ core/src/main/java/org/apache/accumulo/core/iterators/ server/src/main/java/org/apache/accumulo/server/tabl...
Date Wed, 19 Dec 2012 21:03:52 GMT
Author: kturner
Date: Wed Dec 19 21:03:52 2012
New Revision: 1424106

URL: http://svn.apache.org/viewvc?rev=1424106&view=rev
Log:
ACCUMULO-867 initial checkin of pertable classpath

Added:
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
    accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
      - copied, changed from r1424085, accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java
Removed:
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoader.java
    accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java
Modified:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
    accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
Wed Dec 19 21:03:52 2012
@@ -188,7 +188,7 @@ public class ClientSideIteratorScanner e
         
         @Override
         public void registerSideChannel(final SortedKeyValueIterator<Key,Value> iter)
{}
-      }, false);
+      }, false, null);
     } catch (IOException e) {
       throw new RuntimeException(e);
     }

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java Wed Dec
19 21:03:52 2012
@@ -291,11 +291,11 @@ public enum Property {
       "The Formatter class to apply on results in the shell"),
   TABLE_INTERPRETER_CLASS("table.interepreter", DefaultScanInterpreter.class.getName(), PropertyType.STRING,
       "The ScanInterpreter class to apply on scan arguments in the shell"),
+  TABLE_CLASSPATH("table.classpath.context", "", PropertyType.STRING, "Per table classpath"),
       
       
   //VFS ClassLoader properties
   VFS_CLASSLOADER_PREFIX("classloader.vfs", null, PropertyType.PREFIX, "Properties in this
category affect the VFS ClassLoader"),
-  VFS_CLASSLOADER_ENABLED("classloader.vfs.enabled", "false", PropertyType.BOOLEAN, "Enable/disable
VFS Classloader"),
   VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY("classloader.vfs.context.classpath.system", "",
PropertyType.STRING,
           "Classpath for the system context"),
   VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY("classloader.vfs.context.names", "", PropertyType.STRING,

@@ -303,6 +303,7 @@ public enum Property {
   VFS_CONTEXT_CLASSPATH_PROPERTY("classloader.vfs.context.classpath.", null, PropertyType.PREFIX,
"Classpath for this context");
       
   
+
   private String key, defaultValue, description;
   private PropertyType type;
   
@@ -364,7 +365,7 @@ public enum Property {
     // white list prefixes
     return key.startsWith(Property.TABLE_PREFIX.getKey()) || key.startsWith(Property.TSERV_PREFIX.getKey())
|| key.startsWith(Property.LOGGER_PREFIX.getKey())
         || key.startsWith(Property.MASTER_PREFIX.getKey()) || key.startsWith(Property.GC_PREFIX.getKey())
-        || key.startsWith(Property.MONITOR_PREFIX.getKey() + "banner.");
+        || key.startsWith(Property.MONITOR_PREFIX.getKey() + "banner.") || key.startsWith(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey());
   }
   
   public static Property getPropertyByKey(String key) {

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
Wed Dec 19 21:03:52 2012
@@ -210,12 +210,13 @@ public class IteratorUtil {
       }
     }
     
-    return loadIterators(source, iters, allOptions, env, useAccumuloClassLoader);
+    return loadIterators(source, iters, allOptions, env, useAccumuloClassLoader, conf.get(Property.TABLE_CLASSPATH));
   }
   
   @SuppressWarnings("unchecked")
   public static <K extends WritableComparable<?>,V extends Writable> SortedKeyValueIterator<K,V>
loadIterators(SortedKeyValueIterator<K,V> source,
-      Collection<IterInfo> iters, Map<String,Map<String,String>> iterOpts,
IteratorEnvironment env, boolean useAccumuloClassLoader) throws IOException {
+      Collection<IterInfo> iters, Map<String,Map<String,String>> iterOpts,
IteratorEnvironment env, boolean useAccumuloClassLoader, String context)
+      throws IOException {
     // wrap the source in a SynchronizedIterator in case any of the additional configured
iterators want to use threading
     SortedKeyValueIterator<K,V> prev = new SynchronizedIterator<K,V>(source);
     
@@ -224,7 +225,11 @@ public class IteratorUtil {
        
         Class<? extends SortedKeyValueIterator<K,V>> clazz;
         if (useAccumuloClassLoader){
-          clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloVFSClassLoader.loadClass(iterInfo.className,
SortedKeyValueIterator.class);
+          if (context != null && !context.equals(""))
+            clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloVFSClassLoader.getContextManager().loadClass(context,
iterInfo.className,
+                SortedKeyValueIterator.class);
+          else
+            clazz = (Class<? extends SortedKeyValueIterator<K,V>>) AccumuloVFSClassLoader.loadClass(iterInfo.className,
SortedKeyValueIterator.class);
         }else{
           clazz = (Class<? extends SortedKeyValueIterator<K,V>>) Class.forName(iterInfo.className).asSubclass(SortedKeyValueIterator.class);
         }

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Wed Dec 19 21:03:52 2012
@@ -122,6 +122,7 @@ import org.apache.accumulo.server.util.M
 import org.apache.accumulo.server.util.MetadataTable.LogEntry;
 import org.apache.accumulo.server.util.TabletOperations;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.hadoop.conf.Configuration;
@@ -1469,6 +1470,13 @@ public class Tablet {
           + " entries created)");
     }
     
+    String contextName = acuTableConf.get(Property.TABLE_CLASSPATH);
+    if (contextName != null && !contextName.equals("")) {
+      // initialize context classloader, instead of possibly waiting for it to initialize
for a scan
+      // TODO this could hang causing other tablets to fail to load
+      AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
+    }
+
     // do this last after tablet is completely setup because it
     // could cause major compaction to start
     datafileManager = new DatafileManager(datafiles);

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Wed Dec 19 21:03:52 2012
@@ -198,6 +198,8 @@ import org.apache.accumulo.server.zookee
 import org.apache.accumulo.server.zookeeper.ZooLock;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 import org.apache.accumulo.start.Platform;
+import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.start.classloader.vfs.ContextManager;
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -3052,6 +3054,68 @@ public class TabletServer extends Abstra
       }
     }
     
+    try {
+      AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.ContextConfig()
{
+        
+        @Override
+        public boolean isIsolated(String context) {
+          return false;
+        }
+        
+        @Override
+        public Set<String> getContextURIs(String context) {
+          String key = Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + context;
+          
+          Iterator<Entry<String,String>> iter = getSystemConfiguration().iterator();
+          while (iter.hasNext()) {
+            Entry<String,String> entry = iter.next();
+            if (entry.getKey().equals(key)) {
+              return new HashSet<String>(Arrays.asList(entry.getValue().split(",")));
+            }
+          }
+          
+          return null;
+        }
+      });
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+
+    // A task that cleans up unused classloader contexts
+    TimerTask contextCleaner = new TimerTask() {
+      @Override
+      public void run() {
+        ArrayList<KeyExtent> extents;
+        
+        synchronized (onlineTablets) {
+          extents = new ArrayList<KeyExtent>(onlineTablets.keySet());
+        }
+        
+        Set<Text> tables = new HashSet<Text>();
+        
+        for (KeyExtent keyExtent : extents) {
+          tables.add(keyExtent.getTableId());
+        }
+        
+        HashSet<String> contexts = new HashSet<String>();
+        
+        for (Text tableid : tables) {
+          String context = getTableConfiguration(new KeyExtent(tableid, null, null)).get(Property.TABLE_CLASSPATH);
+          if (!context.equals("")) {
+            contexts.add(context);
+          }
+        }
+        
+        try {
+          AccumuloVFSClassLoader.getContextManager().removeUnusedContexts(contexts);
+        } catch (IOException e) {
+          log.warn(e.getMessage(), e);
+        }
+      }
+    };
+    
+    SimpleTimer.getInstance().schedule(contextCleaner, 60000, 60000);
+
     FileSystemMonitor.start(getSystemConfiguration(), Property.TSERV_MONITOR_FS);
     
     TimerTask gcDebugTask = new TimerTask() {

Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
(original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
Wed Dec 19 21:03:52 2012
@@ -20,7 +20,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 import java.security.SecureClassLoader;
-import java.util.Arrays;
 import java.util.Enumeration;
 
 import org.apache.commons.vfs2.FileChangeEvent;
@@ -154,7 +153,17 @@ public class AccumuloReloadingVFSClassLo
   
   @Override
   public String toString() {
-    return Arrays.toString(this.files);
+    StringBuilder buf = new StringBuilder();
+    
+    for (FileObject f : files) {
+      try {
+        buf.append("\t").append(f.getURL().toString()).append("\n");
+      } catch (FileSystemException e) {
+        log.error("Error getting URL for file", e);
+      }
+    }
+    
+    return buf.toString();
   }
   
 }

Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
(original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
Wed Dec 19 21:03:52 2012
@@ -74,8 +74,6 @@ public class AccumuloVFSClassLoader {
 
   private static final Logger log = Logger.getLogger(AccumuloVFSClassLoader.class);
   
-  public static final String VFS_CLASSLOADER_ENABLED = "classloader.vfs.enabled";
-  
   public static final String VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY = "classloader.vfs.context.classpath.system";
   
   public static final String VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY = "classloader.vfs.context.names";
@@ -84,10 +82,12 @@ public class AccumuloVFSClassLoader {
   
   private static DefaultFileSystemManager vfs = null;
   private static ClassLoader parent = null;
-  private static volatile AccumuloContextClassLoader loader = null;
+  private static volatile AccumuloReloadingVFSClassLoader loader = null;
   private static final Object lock = new Object();
   private static final Configuration ACC_CONF = new Configuration();  
   private static final String SITE_CONF;
+  private static ContextManager contextManager;
+
   static {
     String configFile = System.getProperty("org.apache.accumulo.config.file", "accumulo-site.xml");
     if (System.getenv("ACCUMULO_HOME") != null) {
@@ -230,11 +230,6 @@ public class AccumuloVFSClassLoader {
       synchronized (lock) {
         if (null == loader) {
           
-          if (ACC_CONF.getBoolean(VFS_CLASSLOADER_ENABLED, false) == false) {
-            localLoader = AccumuloClassLoader.getClassLoader();
-            return localLoader;
-          }
-          
           if (null == vfs) {
             vfs = new DefaultFileSystemManager();
             //TODO: Might be able to use a different cache impl or specify cache directory
in configuration.
@@ -276,18 +271,18 @@ public class AccumuloVFSClassLoader {
             vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
             vfs.init();
           }
-                    
-          //Set up the 2nd tier class loader
+          
+          // Set up the 2nd tier class loader
           if (null == parent)
             parent = getAccumuloClassLoader();
           
-          //Get the default context classpaths from the configuration
+          // Get the default context classpaths from the configuration
           String[] defaultPaths = ACC_CONF.getStrings(VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY);
           if (null == defaultPaths || defaultPaths.length == 0) {
-            log.info("Default context not configured.");
             localLoader = parent;
             return localLoader;
           }
+
           ArrayList<FileObject> defaultClassPath = new ArrayList<FileObject>();
           for (String path : defaultPaths) {
             FileObject fo = vfs.resolveFile(path);
@@ -301,30 +296,7 @@ public class AccumuloVFSClassLoader {
           }
           
           //Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
-          loader = new AccumuloContextClassLoader(defaultClassPath.toArray(new FileObject[defaultClassPath.size()]),
vfs, parent);
-
-          //Add the other contexts
-          String[] contexts = ACC_CONF.getStrings(VFS_CLASSLOADER_CONTEXT_NAMES_PROPERTY);
-          if (null != contexts) {
-            for (String context : contexts) {
-              String[] contextPaths = ACC_CONF.getStrings(VFS_CONTEXT_CLASSPATH_PROPERTY
+ context);
-              if (null != contextPaths) {
-                ArrayList<FileObject> contextClassPath = new ArrayList<FileObject>();
-                for (String cp : contextPaths) {
-                  FileObject fo = vfs.resolveFile(cp);
-                  if (fo.getType().equals(FileType.FILE)) {
-                    contextClassPath.add(fo);
-                  } else {
-                    for (FileObject child : fo.getChildren()) {
-                      contextClassPath.add(child);                
-                    }                    
-                  }
-                }
-                log.debug("Creating Context ClassLoader for context: " + context + " using
paths: " + contextClassPath.toString());
-                loader.addContext(context, contextClassPath.toArray(new FileObject[contextClassPath.size()]));
-              }
-            }
-          }
+          loader = new AccumuloReloadingVFSClassLoader(defaultClassPath.toArray(new FileObject[defaultClassPath.size()]),
vfs, parent);
         }
       }
       localLoader = loader;
@@ -335,18 +307,20 @@ public class AccumuloVFSClassLoader {
   public static void printClassPath() {
     try {
       ClassLoader cl = getClassLoader();
-      if (ACC_CONF.getBoolean(VFS_CLASSLOADER_ENABLED, false) == false) {
-        //If using older classloader, then use its printClassPath method
-        AccumuloClassLoader.printClassPath();
-      } else if (cl instanceof URLClassLoader) {
+      if (cl instanceof URLClassLoader) {
         //If VFS class loader enabled, but no contexts defined.
         URLClassLoader ucl = (URLClassLoader) cl;
         System.out.println("URL classpath items are: \n");
         for (URL u : ucl.getURLs()) {
-          System.out.println(u.toExternalForm());
+          System.out.println("\t" + u.toExternalForm());
         }
-      } else if (cl instanceof AccumuloContextClassLoader) {
+      } else if (cl instanceof AccumuloReloadingVFSClassLoader) {
         //If VFS class loader enabled and contexts are defined
+        System.out.println("URL classpath items are: \n");
+        URLClassLoader ucl = (URLClassLoader) cl.getParent();
+        for (URL u : ucl.getURLs()) {
+          System.out.println("\t" + u.toExternalForm());
+        }
         System.out.println("VFS classpaths items are:\n" + getClassLoader().toString());
       } else {
         System.out.println("Unknown classloader configuration");
@@ -357,9 +331,18 @@ public class AccumuloVFSClassLoader {
   }
 
   
+  public static synchronized ContextManager getContextManager() throws IOException {
+    // TODO is there problem with using this lck?
+    
+    if (contextManager == null) {
+      contextManager = new ContextManager(vfs, getClassLoader());
+    }
+
+    return contextManager;
+  }
+
   public static void close() {
     if (null != vfs)
       vfs.close();
   }
-  
 }

Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java?rev=1424106&view=auto
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
(added)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
Wed Dec 19 21:03:52 2012
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.FileType;
+
+public class ContextManager {
+  
+  // there is a lock per context so that one context can initialize w/o blocking another
context
+  private class Context {
+    AccumuloReloadingVFSClassLoader loader;
+    Set<String> uris;
+    boolean closed = false;
+    
+    Context(Set<String> uris) {
+      this.uris = new HashSet<String>(uris);
+    }
+    
+    synchronized AccumuloReloadingVFSClassLoader getClassLoader() throws FileSystemException
{
+      if (closed)
+        return null;
+
+      if (loader == null) {
+        ArrayList<FileObject> defaultClassPath = new ArrayList<FileObject>();
+        for (String path : uris) {
+          FileObject fo = vfs.resolveFile(path);
+          if (fo.getType().equals(FileType.FILE)) {
+            defaultClassPath.add(fo);
+          } else {
+            for (FileObject child : fo.getChildren()) {
+              defaultClassPath.add(child);
+            }
+          }
+        }
+        
+        loader = new AccumuloReloadingVFSClassLoader(defaultClassPath.toArray(new FileObject[0]),
vfs, parent);
+      }
+      
+      return loader;
+    }
+    
+    synchronized void close() {
+      closed = true;
+      loader.close();
+      loader = null;
+    }
+  }
+
+  private Map<String,Context> contexts = new HashMap<String,Context>();
+
+  private volatile ContextConfig config;
+  private FileSystemManager vfs;
+  private ClassLoader parent;
+  
+  ContextManager(FileSystemManager vfs, ClassLoader parent) {
+    this.vfs = vfs;
+    this.parent = parent;
+  }
+  
+  public interface ContextConfig {
+    Set<String> getContextURIs(String context);
+    
+    boolean isIsolated(String context);
+  }
+  
+  /**
+   * configuration must be injected for ContextManager to work
+   * 
+   * @param config
+   */
+  public synchronized void setContextConfig(ContextConfig config) {
+    if (this.config != null)
+      throw new IllegalStateException("Context manager config already set");
+    this.config = config;
+  }
+  
+  public ClassLoader getClassLoader(String contextName) throws FileSystemException {
+
+    Set<String> uris = config.getContextURIs(contextName);
+    
+    if (uris == null)
+      throw new IllegalArgumentException("Unknown context " + contextName);
+    
+    Context context = null;
+    Context contextToClose = null;
+    
+    synchronized (this) {
+      // only manipulate internal data structs in this sync block... avoid creating or closing
classloader, reading config, etc... basically avoid operations
+      // that may block
+      context = contexts.get(context);
+      
+      if (context == null) {
+        context = new Context(uris);
+        contexts.put(contextName, context);
+      } else if (!context.uris.equals(uris)) {
+        contextToClose = context;
+        context = new Context(uris);
+        contexts.put(contextName, context);
+      }
+    }
+    
+    if (contextToClose != null)
+      contextToClose.close();
+
+    AccumuloReloadingVFSClassLoader loader = context.getClassLoader();
+    if (loader == null) {
+      // ooppss, context was closed by another thread, try again
+      return getClassLoader(contextName);
+    }
+    
+    return loader;
+
+  }
+  
+  public <U> Class<? extends U> loadClass(String context, String classname, Class<U>
extension) throws ClassNotFoundException {
+    try {
+      return (Class<? extends U>) getClassLoader(context).loadClass(classname).asSubclass(extension);
+    } catch (IOException e) {
+      throw new ClassNotFoundException("IO Error loading class " + classname, e);
+    }
+  }
+  
+  public void removeUnusedContexts(Set<String> inUse) {
+    
+    Map<String,Context> unused;
+    
+    synchronized (this) {
+      unused = new HashMap<String,Context>(contexts);
+      unused.keySet().removeAll(inUse);
+      contexts.keySet().removeAll(unused.keySet());
+    }
+    
+    for (Context context : unused.values()) {
+      // close outside of lock
+      context.close();
+    }
+  }
+}

Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java?rev=1424106&r1=1424105&r2=1424106&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
(original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoaderTest.java
Wed Dec 19 21:03:52 2012
@@ -51,7 +51,7 @@ public class AccumuloVFSClassLoaderTest 
     Whitebox.setInternalState(AccumuloVFSClassLoader.class, "log", Logger.getLogger(AccumuloVFSClassLoader.class));
     ClassLoader acl = AccumuloVFSClassLoader.getClassLoader();
     Assert.assertTrue((acl instanceof URLClassLoader));
-    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader)
null);
+    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader)
null);
   }
   
   /*
@@ -67,7 +67,7 @@ public class AccumuloVFSClassLoaderTest 
     Whitebox.setInternalState(AccumuloVFSClassLoader.class, "log", Logger.getLogger(AccumuloVFSClassLoader.class));
     ClassLoader acl = AccumuloVFSClassLoader.getClassLoader();
     Assert.assertTrue((acl instanceof URLClassLoader));
-    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader)
null);
+    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader)
null);
     // URLClassLoader ucl = (URLClassLoader) acl;
     // URL[] classpath = ucl.getURLs();
     // System.out.println(Arrays.toString(classpath));
@@ -97,71 +97,19 @@ public class AccumuloVFSClassLoaderTest 
     Whitebox.setInternalState(AccumuloVFSClassLoader.class, "lock", new Object());
     Whitebox.setInternalState(AccumuloVFSClassLoader.class, "log", Logger.getLogger(AccumuloVFSClassLoader.class));
     ClassLoader acl = AccumuloVFSClassLoader.getClassLoader();
-    Assert.assertTrue((acl instanceof AccumuloContextClassLoader));
-    AccumuloContextClassLoader accl = (AccumuloContextClassLoader) acl;
-    AccumuloReloadingVFSClassLoader arvcl = accl.getClassLoader(AccumuloContextClassLoader.DEFAULT_CONTEXT);
+    Assert.assertTrue((acl instanceof AccumuloReloadingVFSClassLoader));
+    AccumuloReloadingVFSClassLoader arvcl = (AccumuloReloadingVFSClassLoader) acl;
     Assert.assertEquals(1, arvcl.getFiles().length);
     Assert.assertTrue(arvcl.getFiles()[0].getURL().toString().equals(AccumuloDFSBase.HDFS_URI
+ "/accumulo/classpath/HelloWorld.jar"));
     Class<?> clazz1 = arvcl.loadClass("test.HelloWorld");
     Object o1 = clazz1.newInstance();
     Assert.assertEquals("Hello World!", o1.toString());
-    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader)
null);
+    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader)
null);
     
     hdfs.delete(DEFAULT, true);
     
   }
-  
-  @Test
-  public void testAdditionalContextConfigured() throws Exception {
-    
-    // Create default and application1 context directory
-    FileSystem hdfs = cluster.getFileSystem();
-    Path DEFAULT = new Path("/accumulo/classpath");
-    hdfs.mkdirs(DEFAULT);
-    Path APPLICATION = new Path("/application1/classpath");
-    hdfs.mkdirs(APPLICATION);
-    
-    // Copy jar file to DEFAULT and APPLICATION directories
-    URL jarPath = this.getClass().getResource("/HelloWorld.jar");
-    Path src = new Path(jarPath.toURI().toString());
-    Path dst = new Path(DEFAULT, src.getName());
-    hdfs.copyFromLocalFile(src, dst);
-    dst = new Path(APPLICATION, src.getName());
-    hdfs.copyFromLocalFile(src, dst);
-    
-    URL defaultDir = this.getClass().getResource("/application1");
-    Configuration conf = new Configuration(hdfs.getConf());
-    conf.addResource(new File(defaultDir.getPath() + "/conf/accumulo-site.xml").toURI().toURL());
-    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "ACC_CONF", conf);
-    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "lock", new Object());
-    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "log", Logger.getLogger(AccumuloVFSClassLoader.class));
-    
-    ClassLoader acl = AccumuloVFSClassLoader.getClassLoader();
-    Assert.assertTrue((acl instanceof AccumuloContextClassLoader));
-    AccumuloContextClassLoader accl = (AccumuloContextClassLoader) acl;
-    // DEFAULT CONTEXT
-    AccumuloReloadingVFSClassLoader arvcl = accl.getClassLoader(AccumuloContextClassLoader.DEFAULT_CONTEXT);
-    Assert.assertEquals(1, arvcl.getFiles().length);
-    Assert.assertTrue(arvcl.getFiles()[0].getURL().toString().equals(AccumuloDFSBase.HDFS_URI
+ "/accumulo/classpath/HelloWorld.jar"));
-    Class<?> clazz1 = arvcl.loadClass("test.HelloWorld");
-    Object o1 = clazz1.newInstance();
-    Assert.assertEquals("Hello World!", o1.toString());
-    
-    // APPLICATION CONTEXT
-    AccumuloReloadingVFSClassLoader arvcl2 = accl.getClassLoader("application1");
-    Assert.assertEquals(1, arvcl2.getFiles().length);
-    Assert.assertTrue(arvcl2.getFiles()[0].getURL().toString().equals(AccumuloDFSBase.HDFS_URI
+ "/application1/classpath/HelloWorld.jar"));
-    Class<?> clazz2 = arvcl2.loadClass("test.HelloWorld");
-    Object o2 = clazz2.newInstance();
-    Assert.assertEquals("Hello World!", o2.toString());
-    
-    Assert.assertTrue(!o1.equals(o2));
-    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader)
null);
-    
-    hdfs.delete(DEFAULT, true);
-    hdfs.delete(APPLICATION, true);
-    
-  }
+
   
   @Test
   public void testLoadClass() throws Exception {
@@ -191,7 +139,7 @@ public class AccumuloVFSClassLoaderTest 
     Object o1 = clazz1.newInstance();
     Assert.assertEquals("Hello World!", o1.toString());
     
-    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloContextClassLoader)
null);
+    Whitebox.setInternalState(AccumuloVFSClassLoader.class, "loader", (AccumuloReloadingVFSClassLoader)
null);
     
     hdfs.delete(DEFAULT, true);
     hdfs.delete(APPLICATION, true);

Copied: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
(from r1424085, accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java)
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java?p2=accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java&p1=accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java&r1=1424085&r2=1424106&rev=1424106&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloContextClassLoaderTest.java
(original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
Wed Dec 19 21:03:52 2012
@@ -17,7 +17,11 @@
 package org.apache.accumulo.start.classloader.vfs;
 
 import java.net.URL;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
 
+import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
 import org.apache.accumulo.test.AccumuloDFSBase;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.hadoop.fs.FileSystem;
@@ -27,13 +31,12 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-public class AccumuloContextClassLoaderTest extends AccumuloDFSBase {
+public class ContextManagerTest extends AccumuloDFSBase {
   
   private static final Path TEST_DIR = new Path(HDFS_URI + "/test-dir");
   private static final Path TEST_DIR2 = new Path(HDFS_URI + "/test-dir2");
 
   private FileSystem hdfs = null;
-  private AccumuloContextClassLoader cl = null;
 
   @Before
   public void setup() throws Exception {
@@ -54,33 +57,54 @@ public class AccumuloContextClassLoaderT
 
   @Test
   public void differentContexts() throws Exception {
+    
+    ContextManager cm = new ContextManager(vfs, ClassLoader.getSystemClassLoader());
+    cm.setContextConfig(new ContextConfig() {
+      @Override
+      public Set<String> getContextURIs(String context) {
+        if (context.equals("CX1")) {
+          return new HashSet<String>(Arrays.asList(new Path(TEST_DIR, "HelloWorld.jar").toUri().toString()));
+        } else if (context.equals("CX2")) {
+          return new HashSet<String>(Arrays.asList(new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString()));
+        }
+        return null;
+      }
+      
+      @Override
+      public boolean isIsolated(String context) {
+        // TODO Auto-generated method stub
+        return false;
+      }
+    });
+
     FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString());
     FileObject[] dirContents = testDir.getChildren();
-    cl = new AccumuloContextClassLoader(dirContents, vfs, ClassLoader.getSystemClassLoader());
-    FileObject[] files = cl.getClassLoader(AccumuloContextClassLoader.DEFAULT_CONTEXT).getFiles();
+    ClassLoader cl1 = cm.getClassLoader("CX1");
+    FileObject[] files = ((AccumuloReloadingVFSClassLoader) cl1).getFiles();
     Assert.assertArrayEquals(dirContents, files);
 
+
     FileObject testDir2 = vfs.resolveFile(TEST_DIR2.toUri().toString());
     FileObject[] dirContents2 = testDir2.getChildren();
-    cl.addContext("MYCONTEXT", dirContents2);
-    FileObject[] files2 = cl.getClassLoader("MYCONTEXT").getFiles();
+    ClassLoader cl2 = cm.getClassLoader("CX2");
+    FileObject[] files2 = ((AccumuloReloadingVFSClassLoader) cl2).getFiles();
     Assert.assertArrayEquals(dirContents2, files2);
     
-    Class<?> defaultContextClass = cl.loadClass("test.HelloWorld");
+    Class<?> defaultContextClass = cl1.loadClass("test.HelloWorld");
     Object o1 = defaultContextClass.newInstance();
     Assert.assertEquals("Hello World!", o1.toString());
 
-    Class<?> myContextClass = cl.loadClass("MYCONTEXT", "test.HelloWorld");
+    Class<?> myContextClass = cl2.loadClass("test.HelloWorld");
     Object o2 = myContextClass.newInstance();
     Assert.assertEquals("Hello World!", o2.toString());
     
     Assert.assertFalse(defaultContextClass.equals(myContextClass));
 
+    cm.removeUnusedContexts(new HashSet<String>());
   }
   
   @After
   public void tearDown() throws Exception {
-    cl.close();
     this.hdfs.delete(TEST_DIR, true);
   }
 



Mime
View raw message