Return-Path: Delivered-To: apmail-jakarta-commons-dev-archive@www.apache.org Received: (qmail 27993 invoked from network); 7 Jul 2004 20:01:41 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (209.237.227.199) by minotaur-2.apache.org with SMTP; 7 Jul 2004 20:01:41 -0000 Received: (qmail 19923 invoked by uid 500); 7 Jul 2004 20:01:44 -0000 Delivered-To: apmail-jakarta-commons-dev-archive@jakarta.apache.org Received: (qmail 19823 invoked by uid 500); 7 Jul 2004 20:01:44 -0000 Mailing-List: contact commons-dev-help@jakarta.apache.org; run by ezmlm Precedence: bulk List-Unsubscribe: List-Subscribe: List-Help: List-Post: List-Id: "Jakarta Commons Developers List" Reply-To: "Jakarta Commons Developers List" Delivered-To: mailing list commons-dev@jakarta.apache.org Received: (qmail 19810 invoked by uid 500); 7 Jul 2004 20:01:43 -0000 Received: (qmail 19807 invoked by uid 99); 7 Jul 2004 20:01:43 -0000 X-ASF-Spam-Status: No, hits=0.5 required=10.0 tests=ALL_TRUSTED,NO_REAL_NAME X-Spam-Check-By: apache.org Received: from [209.237.227.194] (HELO minotaur.apache.org) (209.237.227.194) by apache.org (qpsmtpd/0.27.1) with SMTP; Wed, 07 Jul 2004 13:01:43 -0700 Received: (qmail 27940 invoked by uid 1850); 7 Jul 2004 20:01:35 -0000 Date: 7 Jul 2004 20:01:35 -0000 Message-ID: <20040707200135.27939.qmail@minotaur.apache.org> From: imario@apache.org To: jakarta-commons-sandbox-cvs@apache.org Subject: cvs commit: jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/cache SoftRefFilesCache.java X-Virus-Checked: Checked X-Spam-Rating: minotaur-2.apache.org 1.6.2 0/1000/N imario 2004/07/07 13:01:35 Modified: vfs/src/java/org/apache/commons/vfs/provider AbstractFileObject.java AbstractFileSystem.java vfs/src/test/org/apache/commons/vfs/test AbstractProviderTestConfig.java AbstractTestSuite.java NamingTests.java vfs/src/java/org/apache/commons/vfs/impl DefaultFileSystemManager.java vfs/src/java/org/apache/commons/vfs FileSystemManager.java vfs/src/java/org/apache/commons/vfs/cache SoftRefFilesCache.java Log: SoftRefFilesCache Close filesystem if there are no longer files cached for it. This automatically closes the communication link to the underlaying filesystem. Revision Changes Path 1.50 +61 -12 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/provider/AbstractFileObject.java Index: AbstractFileObject.java =================================================================== RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/provider/AbstractFileObject.java,v retrieving revision 1.49 retrieving revision 1.50 diff -u -r1.49 -r1.50 --- AbstractFileObject.java 4 Jul 2004 18:45:55 -0000 1.49 +++ AbstractFileObject.java 7 Jul 2004 20:01:35 -0000 1.50 @@ -58,7 +58,8 @@ public abstract class AbstractFileObject implements FileObject { - private static final FileObject[] EMPTY_FILE_ARRAY = {}; + // private static final FileObject[] EMPTY_FILE_ARRAY = {}; + private static final FileName[] EMPTY_FILE_ARRAY = {}; private final FileName name; private final AbstractFileSystem fs; @@ -68,7 +69,11 @@ private boolean attached; private FileType type; private AbstractFileObject parent; - private FileObject[] children; + + // Changed to hold only the name of the children and let the object + // go into the global files cache + // private FileObject[] children; + private FileName[] children; protected AbstractFileObject(final FileName name, final AbstractFileSystem fs) @@ -511,22 +516,24 @@ // Use cached info, if present if (children != null) { - return children; + return resolveFiles(children); } // allow the filesystem to return resolved children. e.g. prefill type for webdav + FileObject[] childrenObjects; try { - children = doListChildrenResolved(); + childrenObjects = doListChildrenResolved(); + children = extractNames(childrenObjects); } catch (Exception exc) { throw new FileSystemException("vfs.provider/list-children.error", new Object[]{name}, exc); } - if (children != null) + if (childrenObjects != null) { - return children; + return childrenObjects; } // List the children @@ -548,15 +555,54 @@ else { // Create file objects for the children - children = new FileObject[files.length]; + // children = new FileObject[files.length]; + children = new FileName[files.length]; for (int i = 0; i < files.length; i++) { final String file = files[i]; - children[i] = fs.resolveFile(name.resolveName(file, NameScope.CHILD)); + // children[i] = fs.resolveFile(name.resolveName(file, NameScope.CHILD)); + children[i] = name.resolveName(file, NameScope.CHILD); } } - return children; + return resolveFiles(children); + } + + private FileName[] extractNames(FileObject[] objects) + { + if (objects == null) + { + return null; + } + + FileName names[] = new FileName[objects.length]; + for (int iterObjects = 0; iterObjects < objects.length; iterObjects++) + { + names[iterObjects] = objects[iterObjects].getName(); + } + + return names; + } + + private FileObject[] resolveFiles(FileName[] children) throws FileSystemException + { + if (children == null) + { + return null; + } + + FileObject objects[] = new FileObject[children.length]; + for (int iterChildren = 0; iterChildren < children.length; iterChildren++) + { + objects[iterChildren] = resolveFile(children[iterChildren]); + } + + return objects; + } + + private FileObject resolveFile(FileName child) throws FileSystemException + { + return fs.resolveFile(child); } /** @@ -568,11 +614,13 @@ getChildren(); for (int i = 0; i < children.length; i++) { - final FileObject child = children[i]; + // final FileObject child = children[i]; + final FileName child = children[i]; // TODO - use a comparator to compare names - if (child.getName().getBaseName().equals(name)) + // if (child.getName().getBaseName().equals(name)) + if (child.getBaseName().equals(name)) { - return child; + return resolveFile(child); } } return null; @@ -1095,6 +1143,7 @@ { attached = false; type = null; + parent = null; fs.fileDetached(this); 1.29 +6 -10 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/provider/AbstractFileSystem.java Index: AbstractFileSystem.java =================================================================== RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/provider/AbstractFileSystem.java,v retrieving revision 1.28 retrieving revision 1.29 diff -u -r1.28 -r1.29 --- AbstractFileSystem.java 4 Jul 2004 18:45:55 -0000 1.28 +++ AbstractFileSystem.java 7 Jul 2004 20:01:35 -0000 1.29 @@ -52,7 +52,7 @@ private final FileName rootName; private FileObject parentLayer; - private FileObject root; + // private FileObject root; private final Collection caps = new HashSet(); /** @@ -79,6 +79,7 @@ final FileObject parentLayer, final FileSystemOptions fileSystemOptions) { + // this.parentLayer = parentLayer; this.parentLayer = parentLayer; this.rootName = rootName; this.fileSystemOptions = fileSystemOptions; @@ -99,15 +100,7 @@ */ public void close() { - try - { - closeCommunicationLink(); - } - finally - { - // Clean-up - files.clear(this); - } + closeCommunicationLink(); } /** @@ -228,11 +221,14 @@ */ public FileObject getRoot() throws FileSystemException { + return resolveFile(rootName); + /* if (root == null) { root = resolveFile(rootName); } return root; + */ } /** 1.8 +4 -3 jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/AbstractProviderTestConfig.java Index: AbstractProviderTestConfig.java =================================================================== RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/AbstractProviderTestConfig.java,v retrieving revision 1.7 retrieving revision 1.8 diff -u -r1.7 -r1.8 --- AbstractProviderTestConfig.java 10 May 2004 20:09:44 -0000 1.7 +++ AbstractProviderTestConfig.java 7 Jul 2004 20:01:35 -0000 1.8 @@ -16,7 +16,7 @@ package org.apache.commons.vfs.test; import org.apache.commons.vfs.FilesCache; -import org.apache.commons.vfs.cache.DefaultFilesCache; +import org.apache.commons.vfs.cache.SoftRefFilesCache; import org.apache.commons.vfs.impl.DefaultFileSystemManager; /** @@ -42,7 +42,8 @@ { if (cache == null) { - cache = new DefaultFilesCache(); + // cache = new DefaultFilesCache(); + cache = new SoftRefFilesCache(); } return cache; 1.4 +157 -2 jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/AbstractTestSuite.java Index: AbstractTestSuite.java =================================================================== RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/AbstractTestSuite.java,v retrieving revision 1.3 retrieving revision 1.4 diff -u -r1.3 -r1.4 --- AbstractTestSuite.java 23 Jun 2004 18:17:44 -0000 1.3 +++ AbstractTestSuite.java 7 Jul 2004 20:01:35 -0000 1.4 @@ -27,9 +27,12 @@ import org.apache.commons.vfs.provider.local.DefaultLocalFileProvider; import java.io.File; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.util.ArrayList; import java.util.Enumeration; +import java.util.List; /** * The suite of tests for a file system. @@ -43,7 +46,7 @@ { private final ProviderTestConfig providerConfig; private final String prefix; - private final TestSuite testSuite; + private TestSuite testSuite; private FileObject baseFolder; private FileObject readFolder; @@ -51,6 +54,9 @@ private DefaultFileSystemManager manager; private File tempDir; + private Thread[] startThreadSnapshot; + private Thread[] endThreadSnapshot; + /** * Adds the tests for a file system to this suite. */ @@ -122,6 +128,8 @@ protected void setUp() throws Exception { + startThreadSnapshot = createThreadSnapshot(); + // Locate the temp directory, and clean it up tempDir = AbstractVfsTestCase.getTestDirectory("temp"); checkTempDir("Temp dir not empty before test"); @@ -167,6 +175,51 @@ protected void tearDown() throws Exception { + readFolder.close(); + writeFolder.close(); + baseFolder.close(); + + readFolder = null; + writeFolder = null; + baseFolder = null; + testSuite = null; + fTest = null; + + // force the SoftRefFilesChache to free all files + System.err.println("."); + System.gc(); + Thread.sleep(1000); + System.err.println("."); + System.gc(); + Thread.sleep(1000); + System.err.println("."); + System.gc(); + Thread.sleep(1000); + System.err.println("."); + System.gc(); + Thread.sleep(1000); + + manager.freeUnusedResources(); + endThreadSnapshot = createThreadSnapshot(); + + Thread[] diffThreadSnapshot = diffThreadSnapshot(startThreadSnapshot, endThreadSnapshot); + if (diffThreadSnapshot.length > 0) + { + String message = dumpThreadSnapshot(diffThreadSnapshot); + /* + if (providerConfig.checkCleanThreadState()) + { + // close the manager to do a "not thread safe" release of all resources + // and allow the vm to shutdown + manager.close(); + fail(message); + } + else + { + */ + System.out.println(message); + // } + } manager.close(); // Make sure temp directory is empty or gone @@ -184,4 +237,106 @@ } } + private String dumpThreadSnapshot(Thread[] threadSnapshot) + { + StringBuffer sb = new StringBuffer(256); + sb.append("created threads still running:\n"); + + Field threadTargetField = null; + try + { + threadTargetField = Thread.class.getDeclaredField("target"); + threadTargetField.setAccessible(true); + } + catch (NoSuchFieldException e) + { + ; + } + + for (int iter = 0; iter < threadSnapshot.length; iter++) + { + Thread thread = threadSnapshot[iter]; + if (thread == null) + { + continue; + } + + sb.append("#"); + sb.append(iter + 1); + sb.append(": "); + sb.append(thread.getThreadGroup().getName()); + sb.append("\t"); + sb.append(thread.getName()); + sb.append("\t"); + if (thread.isDaemon()) + { + sb.append("daemon"); + } + else + { + sb.append("not_a_daemon"); + } + + if (threadTargetField != null) + { + sb.append("\t"); + try + { + Object threadTarget = threadTargetField.get(thread); + if (threadTarget != null) + { + sb.append(threadTarget.getClass()); + } + else + { + sb.append("null"); + } + } + catch (IllegalAccessException e) + { + sb.append("unknown class"); + } + } + + sb.append("\n"); + } + + return sb.toString(); + } + + private Thread[] diffThreadSnapshot(Thread[] startThreadSnapshot, Thread[] endThreadSnapshot) + { + List diff = new ArrayList(10); + + nextEnd: for (int iterEnd = 0; iterEnd < endThreadSnapshot.length; iterEnd++) + { + nextStart: for (int iterStart = 0; iterStart < startThreadSnapshot.length; iterStart++) + { + if (startThreadSnapshot[iterStart] == endThreadSnapshot[iterEnd]) + { + continue nextEnd; + } + } + + diff.add(endThreadSnapshot[iterEnd]); + } + + Thread ret[] = new Thread[diff.size()]; + diff.toArray(ret); + return ret; + } + + private Thread[] createThreadSnapshot() + { + ThreadGroup tg = Thread.currentThread().getThreadGroup(); + while (tg.getParent() != null) + { + tg = tg.getParent(); + } + + Thread snapshot[] = new Thread[200]; + tg.enumerate(snapshot, true); + + return snapshot; + } } 1.7 +7 -1 jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/NamingTests.java Index: NamingTests.java =================================================================== RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/test/org/apache/commons/vfs/test/NamingTests.java,v retrieving revision 1.6 retrieving revision 1.7 diff -u -r1.6 -r1.7 --- NamingTests.java 10 May 2004 20:09:44 -0000 1.6 +++ NamingTests.java 7 Jul 2004 20:01:35 -0000 1.7 @@ -53,6 +53,9 @@ // Locate parent file = getManager().resolveFile(".."); assertSame("file object", getReadFolder().getParent(), file); + + // free basefile + getManager().setBaseFile((FileObject) null); } /** @@ -113,6 +116,9 @@ catch (FileSystemException e) { } + + // free basefile + getManager().setBaseFile((FileObject) null); } /** 1.35 +7 -3 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/impl/DefaultFileSystemManager.java Index: DefaultFileSystemManager.java =================================================================== RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/impl/DefaultFileSystemManager.java,v retrieving revision 1.34 retrieving revision 1.35 diff -u -r1.34 -r1.35 --- DefaultFileSystemManager.java 7 Jul 2004 12:03:32 -0000 1.34 +++ DefaultFileSystemManager.java 7 Jul 2004 20:01:35 -0000 1.35 @@ -418,6 +418,7 @@ /** * Sets the base file to use when resolving relative URI. */ + // public void setBaseFile(final FileObject baseFile) public void setBaseFile(final FileObject baseFile) throws FileSystemException { @@ -435,7 +436,7 @@ /** * Returns the base file used to resolve relative URI. */ - public FileObject getBaseFile() + public FileObject getBaseFile() throws FileSystemException { return baseFile; } @@ -445,7 +446,8 @@ */ public FileObject resolveFile(final String uri) throws FileSystemException { - return resolveFile(baseFile, uri); + // return resolveFile(baseFile, uri); + return resolveFile(getBaseFile(), uri); } /** @@ -454,7 +456,8 @@ public FileObject resolveFile(final String uri, final FileSystemOptions fileSystemOptions) throws FileSystemException { - return resolveFile(baseFile, uri, fileSystemOptions); + // return resolveFile(baseFile, uri, fileSystemOptions); + return resolveFile(getBaseFile(), uri, fileSystemOptions); } /** @@ -528,6 +531,7 @@ { throw new FileSystemException("vfs.impl/find-rel-file.error", uri); } + return baseFile.resolveFile(decodedUri); } 1.22 +1 -1 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/FileSystemManager.java Index: FileSystemManager.java =================================================================== RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/FileSystemManager.java,v retrieving revision 1.21 retrieving revision 1.22 diff -u -r1.21 -r1.22 --- FileSystemManager.java 28 Jun 2004 19:44:57 -0000 1.21 +++ FileSystemManager.java 7 Jul 2004 20:01:35 -0000 1.22 @@ -70,7 +70,7 @@ /** * Returns the base file used to resolve relative paths. */ - FileObject getBaseFile(); + FileObject getBaseFile() throws FileSystemException; /** * Locates a file by name. Equivalent to calling 1.7 +65 -12 jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/cache/SoftRefFilesCache.java Index: SoftRefFilesCache.java =================================================================== RCS file: /home/cvs/jakarta-commons-sandbox/vfs/src/java/org/apache/commons/vfs/cache/SoftRefFilesCache.java,v retrieving revision 1.6 retrieving revision 1.7 diff -u -r1.6 -r1.7 --- SoftRefFilesCache.java 21 May 2004 20:54:31 -0000 1.6 +++ SoftRefFilesCache.java 7 Jul 2004 20:01:35 -0000 1.7 @@ -21,6 +21,7 @@ import org.apache.commons.vfs.FileObject; import org.apache.commons.vfs.FileSystem; import org.apache.commons.vfs.VfsLog; +import org.apache.commons.vfs.provider.AbstractFileSystem; import org.apache.commons.vfs.util.Messages; import java.lang.ref.Reference; @@ -46,7 +47,7 @@ */ private Log log = LogFactory.getLog(SoftRefFilesCache.class); - private final Map files = new TreeMap(); + private final Map filesystemCache = new HashMap(); private final Map refReverseMap = new HashMap(100); private final ReferenceQueue refqueue = new ReferenceQueue(); @@ -80,7 +81,10 @@ { FileSystemAndNameKey key = (FileSystemAndNameKey) refReverseMap.get(ref); - removeFile(key); + if (key != null) + { + removeFile(key); + } } } } @@ -122,14 +126,18 @@ { synchronized (this) { + Map files = getOrCreateFilesystemCache(file.getFileSystem()); + /* if (files.size() < 1) { startThread(); } + */ SoftReference ref = new SoftReference(file, refqueue); FileSystemAndNameKey key = new FileSystemAndNameKey(file.getFileSystem(), file.getName()); - files.put(key, ref); + // files.put(key, ref); + files.put(file.getName(), ref); refReverseMap.put(ref, key); } } @@ -138,9 +146,11 @@ { synchronized (this) { - FileSystemAndNameKey key = new FileSystemAndNameKey(filesystem, name); + Map files = getOrCreateFilesystemCache(filesystem); + // FileSystemAndNameKey key = new FileSystemAndNameKey(filesystem, name); - SoftReference ref = (SoftReference) files.get(key); + // SoftReference ref = (SoftReference) files.get(key); + SoftReference ref = (SoftReference) files.get(name); if (ref == null) { return null; @@ -149,7 +159,8 @@ FileObject fo = (FileObject) ref.get(); if (fo == null) { - removeFile(key); + // removeFile(key); + removeFile(filesystem, name); } return fo; } @@ -159,25 +170,40 @@ { synchronized (this) { - Iterator iterKeys = files.keySet().iterator(); + Map files = getOrCreateFilesystemCache(filesystem); + + Iterator iterKeys = refReverseMap.values().iterator(); + // Iterator iterKeys = files.keySet().iterator(); while (iterKeys.hasNext()) { FileSystemAndNameKey key = (FileSystemAndNameKey) iterKeys.next(); if (key.getFileSystem() == filesystem) { - Object ref = files.get(key); + // Object ref = files.get(key); iterKeys.remove(); - refReverseMap.remove(ref); + files.remove(key.getFileName()); + // refReverseMap.remove(ref); } } if (files.size() < 1) { + filesystemClose(filesystem); + } + /* + if (files.size() < 1) + { endThread(); } + */ } } + private void filesystemClose(FileSystem filesystem) + { + ((AbstractFileSystem) filesystem).close(); + } + public void close() { super.close(); @@ -186,7 +212,8 @@ { endThread(); - files.clear(); + // files.clear(); + filesystemCache.clear(); refReverseMap.clear(); } } @@ -204,7 +231,10 @@ { synchronized (this) { - Object ref = files.remove(key); + Map files = getOrCreateFilesystemCache(key.getFileSystem()); + + // Object ref = files.remove(key); + Object ref = files.remove(key.getFileName()); if (ref != null) { refReverseMap.remove(ref); @@ -212,9 +242,32 @@ if (files.size() < 1) { + filesystemClose(key.getFileSystem()); + } + + /* + if (files.size() < 1) + { endThread(); } + */ } } + protected Map getOrCreateFilesystemCache(final FileSystem filesystem) + { + if (filesystemCache.size() < 1) + { + startThread(); + } + + Map files = (Map) filesystemCache.get(filesystem); + if (files == null) + { + files = new TreeMap(); + filesystemCache.put(filesystem, files); + } + + return files; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commons-dev-unsubscribe@jakarta.apache.org For additional commands, e-mail: commons-dev-help@jakarta.apache.org