accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1430092 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/conf/ core/src/main/java/org/apache/accumulo/core/master/thrift/ server/src/main/java/org/apache/accumulo/server/tabletserver/ start/src/main/java/org/apache/accumul...
Date Mon, 07 Jan 2013 23:51:06 GMT
Author: kturner
Date: Mon Jan  7 23:51:06 2013
New Revision: 1430092

URL: http://svn.apache.org/viewvc?rev=1430092&view=rev
Log:
ACCUMULO-869 added post delegation for per table classloaders

Added:
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
Modified:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
    accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
    accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/conf/Property.java Mon Jan
 7 23:51:06 2013
@@ -299,7 +299,12 @@ public enum Property {
   //VFS ClassLoader properties
   VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY(AccumuloVFSClassLoader.VFS_CLASSLOADER_SYSTEM_CLASSPATH_PROPERTY,
"", PropertyType.STRING,
       "Configuration for a system level vfs classloader.  Accumulo jar can be configured
here and loaded out of HDFS."),
-  VFS_CONTEXT_CLASSPATH_PROPERTY(AccumuloVFSClassLoader.VFS_CONTEXT_CLASSPATH_PROPERTY, null,
PropertyType.PREFIX, "Classpath for this context");
+  VFS_CONTEXT_CLASSPATH_PROPERTY(AccumuloVFSClassLoader.VFS_CONTEXT_CLASSPATH_PROPERTY, null,
PropertyType.PREFIX,
+      "Properties in this category are define a classpath. These properties start  with the
category prefix, followed by a context name.  "
+          + "The value is a comma seperated list of URIs. Supports full regex on filename
alone. For example general.vfs.context.classpath.cx1=hdfs://nn1:9902/mylibdir/*.jar.  "
+          + "You can enable post delegation for a context, which will load classes from the
context first instead of the parent first.  "
+          + "Do this by setting general.vfs.context.classpath.<name>.delegation=post,
where <name> is your context name.  "
+          + "If delegation is not specified, it defaults to loading from parent classloader
first.");
       
   
 

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java
Mon Jan  7 23:51:06 2013
@@ -1127,7 +1127,7 @@ import org.slf4j.LoggerFactory;
                 for (int _i1 = 0; _i1 < _map0.size; ++_i1)
                 {
                   String _key2; // required
-                  TableInfo _val3; // optional
+                  TableInfo _val3; // required
                   _key2 = iprot.readString();
                   _val3 = new TableInfo();
                   _val3.read(iprot);

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Mon Jan  7 23:51:06 2013
@@ -200,6 +200,7 @@ import org.apache.accumulo.server.zookee
 import org.apache.accumulo.start.Platform;
 import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
 import org.apache.accumulo.start.classloader.vfs.ContextManager;
+import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -3059,25 +3060,29 @@ public class TabletServer extends Abstra
     }
     
     try {
-      AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.ContextConfig()
{
-        
-        @Override
-        public boolean isIsolated(String context) {
-          return false;
-        }
-        
+      AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.ContextsConfig()
{
         @Override
-        public String getContextURIs(String context) {
+        public ContextConfig getContextConfig(String context) {
           String key = Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + context;
           
+          String uris = null;
+          boolean preDelegate = true;
+
           Iterator<Entry<String,String>> iter = getSystemConfiguration().iterator();
           while (iter.hasNext()) {
             Entry<String,String> entry = iter.next();
             if (entry.getKey().equals(key)) {
-              return entry.getValue();
+              uris = entry.getValue();
+            }
+            
+            if (entry.getKey().equals(key + ".delegation") && entry.getValue().trim().equalsIgnoreCase("post"))
{
+              preDelegate = false;
             }
           }
           
+          if (uris != null)
+            return new ContextConfig(uris, preDelegate);
+
           return null;
         }
       });

Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
(original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoader.java
Mon Jan  7 23:51:06 2013
@@ -46,13 +46,19 @@ public class AccumuloReloadingVFSClassLo
   private ReloadingClassLoader parent = null;
   private DefaultFileMonitor monitor = null;
   private VFSClassLoader cl = null;
+  private boolean preDelegate;
 
   @Override
   public synchronized ClassLoader getClassLoader() {
     if (cl == null || cl.getParent() != parent.getClassLoader()) {
       try {
         files = AccumuloVFSClassLoader.resolve(vfs, uris);
-        setClassloader(new VFSClassLoader(files, vfs, parent.getClassLoader()));
+        
+        if (preDelegate)
+          cl = new VFSClassLoader(files, vfs, parent.getClassLoader());
+        else
+          cl = new PostDelegatingVFSClassLoader(files, vfs, parent.getClassLoader());
+
       } catch (FileSystemException fse) {
         throw new RuntimeException(fse);
       }
@@ -66,16 +72,21 @@ public class AccumuloReloadingVFSClassLo
     
   }
 
-  public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs, ReloadingClassLoader
parent, long monitorDelay) throws FileSystemException {
+  public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs, ReloadingClassLoader
parent, long monitorDelay, boolean preDelegate)
+      throws FileSystemException {
 
     this.uris = uris;
     this.vfs = vfs;
     this.parent = parent;
+    this.preDelegate = preDelegate;
     
     ArrayList<FileObject> pathsToMonitor = new ArrayList<FileObject>();
     files = AccumuloVFSClassLoader.resolve(vfs, uris, pathsToMonitor);
 
-    cl = new VFSClassLoader(files, vfs, parent.getClassLoader());
+    if (preDelegate)
+      cl = new VFSClassLoader(files, vfs, parent.getClassLoader());
+    else
+      cl = new PostDelegatingVFSClassLoader(files, vfs, parent.getClassLoader());
     
     monitor = new DefaultFileMonitor(this);
     monitor.setDelay(monitorDelay);
@@ -87,8 +98,9 @@ public class AccumuloReloadingVFSClassLo
     monitor.start();
   }
   
-  public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs, final ReloadingClassLoader
parent) throws FileSystemException {
-    this(uris, vfs, parent, DEFAULT_TIMEOUT);
+  public AccumuloReloadingVFSClassLoader(String uris, FileSystemManager vfs, final ReloadingClassLoader
parent, boolean preDelegate)
+      throws FileSystemException {
+    this(uris, vfs, parent, DEFAULT_TIMEOUT, preDelegate);
   }
 
   public FileObject[] getFiles() {

Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
(original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
Mon Jan  7 23:51:06 2013
@@ -175,7 +175,7 @@ public class AccumuloVFSClassLoader {
       return wrapper;
 
     // TODO monitor time for lib/ext was 1 sec... should this be configurable?
-    return new AccumuloReloadingVFSClassLoader(dynamicCPath, vfs, wrapper, 1000);
+    return new AccumuloReloadingVFSClassLoader(dynamicCPath, vfs, wrapper, 1000, true);
   }
 
   public static ClassLoader getClassLoader() throws IOException {

Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
(original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/ContextManager.java
Mon Jan  7 23:51:06 2013
@@ -29,22 +29,22 @@ public class ContextManager {
   // there is a lock per context so that one context can initialize w/o blocking another
context
   private class Context {
     AccumuloReloadingVFSClassLoader loader;
-    String uris;
+    ContextConfig cconfig;
     boolean closed = false;
     
-    Context(String uris) {
-      this.uris = uris;
+    Context(ContextConfig cconfig) {
+      this.cconfig = cconfig;
     }
     
-    synchronized AccumuloReloadingVFSClassLoader getClassLoader() throws FileSystemException
{
+    synchronized ClassLoader getClassLoader() throws FileSystemException {
       if (closed)
         return null;
       
       if (loader == null) {
-        loader = new AccumuloReloadingVFSClassLoader(uris, vfs, parent);
+        loader = new AccumuloReloadingVFSClassLoader(cconfig.uris, vfs, parent, cconfig.preDelegation);
       }
       
-      return loader;
+      return loader.getClassLoader();
     }
     
     synchronized void close() {
@@ -56,7 +56,7 @@ public class ContextManager {
 
   private Map<String,Context> contexts = new HashMap<String,Context>();
 
-  private volatile ContextConfig config;
+  private volatile ContextsConfig config;
   private FileSystemManager vfs;
   private ReloadingClassLoader parent;
   
@@ -65,10 +65,28 @@ public class ContextManager {
     this.parent = parent;
   }
   
-  public interface ContextConfig {
-    String getContextURIs(String context);
+  public static class ContextConfig {
+    String uris;
+    boolean preDelegation;
+    
+    public ContextConfig(String uris, boolean preDelegation) {
+      this.uris = uris;
+      this.preDelegation = preDelegation;
+    }
     
-    boolean isIsolated(String context);
+    public boolean equals(Object o) {
+      if (o instanceof ContextConfig) {
+        ContextConfig oc = (ContextConfig) o;
+        
+        return uris.equals(oc.uris) && preDelegation == oc.preDelegation;
+      }
+      
+      return false;
+    }
+  }
+  
+  public interface ContextsConfig {
+    ContextConfig getContextConfig(String context);
   }
   
   /**
@@ -76,7 +94,7 @@ public class ContextManager {
    * 
    * @param config
    */
-  public synchronized void setContextConfig(ContextConfig config) {
+  public synchronized void setContextConfig(ContextsConfig config) {
     if (this.config != null)
       throw new IllegalStateException("Context manager config already set");
     this.config = config;
@@ -84,9 +102,9 @@ public class ContextManager {
   
   public ClassLoader getClassLoader(String contextName) throws FileSystemException {
 
-    String uris = config.getContextURIs(contextName);
+    ContextConfig cconfig = config.getContextConfig(contextName);
     
-    if (uris == null)
+    if (cconfig == null)
       throw new IllegalArgumentException("Unknown context " + contextName);
     
     Context context = null;
@@ -98,11 +116,11 @@ public class ContextManager {
       context = contexts.get(context);
       
       if (context == null) {
-        context = new Context(uris);
+        context = new Context(cconfig);
         contexts.put(contextName, context);
-      } else if (!context.uris.equals(uris)) {
+      } else if (!context.cconfig.equals(cconfig)) {
         contextToClose = context;
-        context = new Context(uris);
+        context = new Context(cconfig);
         contexts.put(contextName, context);
       }
     }
@@ -110,13 +128,13 @@ public class ContextManager {
     if (contextToClose != null)
       contextToClose.close();
 
-    AccumuloReloadingVFSClassLoader loader = context.getClassLoader();
+    ClassLoader loader = context.getClassLoader();
     if (loader == null) {
       // ooppss, context was closed by another thread, try again
       return getClassLoader(contextName);
     }
     
-    return loader.getClassLoader();
+    return loader;
 
   }
   

Added: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java?rev=1430092&view=auto
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
(added)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/classloader/vfs/PostDelegatingVFSClassLoader.java
Mon Jan  7 23:51:06 2013
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.start.classloader.vfs;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.impl.VFSClassLoader;
+
+/**
+ * 
+ */
+public class PostDelegatingVFSClassLoader extends VFSClassLoader {
+  
+  /**
+   * @param files
+   * @param manager
+   * @param parent
+   * @throws FileSystemException
+   */
+  public PostDelegatingVFSClassLoader(FileObject[] files, FileSystemManager manager, ClassLoader
parent) throws FileSystemException {
+    super(files, manager, parent);
+  }
+  
+  protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException
{
+    Class<?> c = findLoadedClass(name);
+    if (c == null) {
+      try {
+        // try finding this class here instead of parent
+        c = findClass(name);
+      } catch (ClassNotFoundException e) {
+
+      }
+    }
+    return super.loadClass(name, resolve);
+  }
+
+}

Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
(original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloReloadingVFSClassLoaderTest.java
Mon Jan  7 23:51:06 2013
@@ -74,7 +74,7 @@ public class AccumuloReloadingVFSClassLo
       public ClassLoader getClassLoader() {
         return ClassLoader.getSystemClassLoader();
       }
-    });
+    }, true);
     
     VFSClassLoader cl = (VFSClassLoader) arvcl.getClassLoader();
     
@@ -94,7 +94,7 @@ public class AccumuloReloadingVFSClassLo
       public ClassLoader getClassLoader() {
         return ClassLoader.getSystemClassLoader();
       }
-    }, 1000);
+    }, 1000, true);
     
     FileObject[] files = ((VFSClassLoader) arvcl.getClassLoader()).getFileObjects();
     Assert.assertArrayEquals(createFileSystems(dirContents), files);

Modified: accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java?rev=1430092&r1=1430091&r2=1430092&view=diff
==============================================================================
--- accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
(original)
+++ accumulo/trunk/start/src/test/java/org/apache/accumulo/start/classloader/vfs/ContextManagerTest.java
Mon Jan  7 23:51:06 2013
@@ -20,6 +20,7 @@ import java.net.URL;
 import java.util.HashSet;
 
 import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextConfig;
+import org.apache.accumulo.start.classloader.vfs.ContextManager.ContextsConfig;
 import org.apache.accumulo.test.AccumuloDFSBase;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
@@ -77,22 +78,16 @@ public class ContextManagerTest extends 
       }
     });
 
-    cm.setContextConfig(new ContextConfig() {
+    cm.setContextConfig(new ContextsConfig() {
       @Override
-      public String getContextURIs(String context) {
+      public ContextConfig getContextConfig(String context) {
         if (context.equals("CX1")) {
-          return new Path(TEST_DIR, "HelloWorld.jar").toUri().toString();
+          return new ContextConfig(new Path(TEST_DIR, "HelloWorld.jar").toUri().toString(),
true);
         } else if (context.equals("CX2")) {
-          return new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString();
+          return new ContextConfig(new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString(),
true);
         }
         return null;
       }
-      
-      @Override
-      public boolean isIsolated(String context) {
-        // TODO Auto-generated method stub
-        return false;
-      }
     });
 
     FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString());
@@ -120,6 +115,35 @@ public class ContextManagerTest extends 
     cm.removeUnusedContexts(new HashSet<String>());
   }
   
+  @Test
+  public void testPostDelegation() throws Exception {
+    final VFSClassLoader parent = new VFSClassLoader(new FileObject[] {vfs.resolveFile(new
Path(TEST_DIR, "HelloWorld.jar").toUri().toString())}, vfs);
+    
+    Class<?> pclass = parent.loadClass("test.HelloWorld");
+    
+    ContextManager cm = new ContextManager(vfs, new ReloadingClassLoader() {
+      @Override
+      public ClassLoader getClassLoader() {
+        return parent;
+      }
+    });
+    
+    cm.setContextConfig(new ContextsConfig() {
+      @Override
+      public ContextConfig getContextConfig(String context) {
+        if (context.equals("CX1")) {
+          return new ContextConfig(new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString(),
true);
+        } else if (context.equals("CX2")) {
+          return new ContextConfig(new Path(TEST_DIR2, "HelloWorld.jar").toUri().toString(),
false);
+        }
+        return null;
+      }
+    });
+    
+    Assert.assertTrue(cm.getClassLoader("CX1").loadClass("test.HelloWorld") == pclass);
+    Assert.assertFalse(cm.getClassLoader("CX2").loadClass("test.HelloWorld") == pclass);
+  }
+
   @After
   public void tearDown() throws Exception {
     this.hdfs.delete(TEST_DIR, true);



Mime
View raw message