commons-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Ken.Tan...@noaa.gov
Subject [vfs] SFTP program not exiting
Date Sun, 23 Mar 2008 02:27:16 GMT
 I'm trying to automate the downloading of data from a remote SFTP
server using VFS. My program gets the files, but doesn't exit, it seems
to hang. It's not clear what cleanup steps I should be doing after
accessing files. I haven't found any VFS + SFTP examples or tutorials.

Below is a maven 2 project file (pom.xml) and an example program
(App.java). Maybe my release() routine need to to have more in it?

If you want to try running the example to see what I'm having trouble
with, the filename pattern is looking for data files with names like
"smoke20070429_wkt.txt". Source, destination and connection information
is hardcoded. Modify the pattern, directories and connection information
for some file on one of your systems.

The pom.xml would go at the top level. The App.java is in a subdirectory;
src/main/java/gov/noaa/eds/byExample/trySftpFileDownload/App.java

Compile with

  mvn assembly:assembly

Execute with

  java -jar
target/trySftpFileDownload-1.0-SNAPSHOT-jar-with-dependencies.jar

If I get a good solution, I can then document a working example on the
VFS wiki.

Thanks,
Ken

=== start pom.xml ===

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>gov.noaa.eds.byExample</groupId>
    <artifactId>trySftpFileDownload</artifactId>
    <packaging>jar</packaging>
    <version>1.0-SNAPSHOT</version>
    <name>trySftpFileDownload</name>
    <url>http://maven.apache.org</url>
    <build>
        <extensions>
            <extension>
                <groupId>org.apache.maven.wagon</groupId>
                <artifactId>wagon-ssh-external</artifactId>
                <version>1.0-beta-2</version>
            </extension>
        </extensions>
        <plugins>
            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.5</source>
                    <target>1.5</target>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                    <archive>
                        <manifest>
                           
<mainClass>gov.noaa.eds.byExample.trySftpFileDownload.App</mainClass>
                        </manifest>
                    </archive>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-antrun-plugin</artifactId>
                <configuration>
                    <tasks>
                        <java
classname="gov.noaa.eds.byExample.trySftpFileDownload.App"
classpathref="maven.runtime.classpath">
                        </java>
                    </tasks>
                </configuration>
            </plugin>
        </plugins>
    </build>
    <dependencies>
        <dependency>
            <groupId>com.jcraft</groupId>
            <artifactId>jsch</artifactId>
            <version>0.1.23</version>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>commons-collections</groupId>
            <artifactId>commons-collections</artifactId>
            <version>3.1</version>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>commons-httpclient</groupId>
            <artifactId>commons-httpclient</artifactId>
            <version>2.0.2</version>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>commons-net</groupId>
            <artifactId>commons-net</artifactId>
            <version>1.4.1</version>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>commons-vfs</groupId>
            <artifactId>commons-vfs</artifactId>
            <version>1.0</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
</project>

=== end pom.xml ===
=== start App.java ===

package gov.noaa.eds.byExample.trySftpFileDownload;

import java.io.File;
import java.util.regex.Pattern;
import org.apache.commons.vfs.AllFileSelector;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSystemException;
import org.apache.commons.vfs.FileSystemManager;
import org.apache.commons.vfs.FileSystemOptions;
import org.apache.commons.vfs.FileType;
import org.apache.commons.vfs.UserAuthenticator;
import org.apache.commons.vfs.VFS;
import org.apache.commons.vfs.auth.StaticUserAuthenticator;
import org.apache.commons.vfs.impl.DefaultFileSystemConfigBuilder;
import org.apache.commons.vfs.provider.local.LocalFile;


/**
 * Example use of sftp
 *
 */
public class App {

    private String localDir = "/extra/data/fires/smoke";
    private File localDirFile;
    private String remoteDir = "/mnt/data1/gdsg/data/fires/prev-smoke";
    // Look for files with names like "smoke20070429_wkt.txt"
    private String filePatternString = ".*smoke\\d{8}_wkt\\.txt";
    private Pattern filePattern;
    private boolean recursive = false;
    private boolean overwrite = false;
    private FileSystemManager fsManager = null;
    private FileSystemOptions opts = null;
    private FileObject sftpFile;
    private String host = "data.source.gov";  // The remote SFTP hostname
    private String user = "smokey";         // Remote system login name
    private String password = "bear";       // Remote system password


    public static void main(String[] args) {
        System.out.println("SFTP download");
        App app = new App();

        app.initialize();
        
        app.process();

        app.release();
    
    } // main( String[] args )


    /**
     * Creates the download directory localDir if it
     * does not exist and makes a connection to the remote SFTP server.
     * 
     */
    public void initialize() {
        if (localDirFile == null) {
            localDirFile = new File(localDir);
        }
        if (!this.localDirFile.exists()) {
            localDirFile.mkdirs();
        }

        try {
            this.fsManager = VFS.getManager();
        } catch (FileSystemException ex) {
            throw new RuntimeException("failed to get fsManager from
VFS", ex);
        }

        UserAuthenticator auth = new StaticUserAuthenticator(null,
this.user,
                this.password);
        this.opts = new FileSystemOptions();
        try {
           
DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(opts,
                    auth);
        } catch (FileSystemException ex) {
            throw new RuntimeException("setUserAuthenticator failed", ex);
        }
        
        this.filePattern = Pattern.compile(filePatternString);
    } // initialize()


    /**
     * Retrieves files that match the specified FileSpec from the SFTP
server
     * and stores them in the work directory.
     */
    public void process() {

        // Set starting path on remote SFTP server.
        try {
            String startPath = "sftp://" + this.host + this.remoteDir;
            this.sftpFile = this.fsManager.resolveFile(startPath, opts);

            System.out.println("SFTP connection successfully established
to " +
                    startPath);
        } catch (FileSystemException ex) {
            throw new RuntimeException("SFTP error parsing path " +
                    this.remoteDir,
                    ex);
        }

        try {
            searchCurrentDirectory("", this.sftpFile);
        } catch (FileSystemException ex) {
            throw new RuntimeException("Error processing remote
directory", ex);
        }
    } // process(Object obj)
    
    
    public void release() {
        try {
            this.sftpFile.close();
        } catch (FileSystemException ex) {
            throw new RuntimeException("Failed to close sftp file", ex);
        }
    } // release()


    /**
     * Search the current working directory of the SFTP client, saving files
     * to the path specified by localDir + the path to the file on the SFTP
     * server. This method will optionally recursively search
directories on 
     * the remote server.
     * 
     * @param path relative path to the local Directory and also corresponds
     * to the relative path from the remote remoteDir
     * @param file remote VFS FileObject directory to examine
     */
    private void searchCurrentDirectory(String path, FileObject file)
throws FileSystemException {
        // Get a directory listing
        FileObject[] children = file.getChildren();

        search:
        for (FileObject f : children) {
            String relativePath = path + File.separatorChar +
                    f.getName().getBaseName();

            System.out.println("relativePath=(.)" + relativePath);

            if (f.getType() == FileType.FOLDER && this.recursive) {
                System.out.println("Recursing into directory " +
f.getName());
                searchCurrentDirectory(relativePath, f);
            } else {
                System.out.println("Examining remote file " + f.getName());
                
                if (!
this.filePattern.matcher(f.getName().getPath()).matches()) {
                    System.out.println("Remote file " + f.getName() +
                            " failed filename pattern check ");
                    continue search;
                }

                boolean getFile = true;
                String localUri = "file://" + this.localDir + relativePath;
                String standardPath = this.localDir + relativePath;
                System.out.println("Standard local path is " +
standardPath);
                LocalFile localFile =
                        (LocalFile) this.fsManager.resolveFile(localUri);
                System.out.println("resolved local file " +
localFile.getName());
                if (localFile.exists()) {
                    if (this.overwrite) {
                        System.out.println("Replacing existing local
file " +
                                standardPath);
                        getFile = true;
                    } else {
                        System.out.println("Skipping download, local
file exists " +
                                standardPath);
                        continue search;
                    }
                } else {
                    getFile = true;
                }

                if (getFile) {
                    if (!localFile.getParent().exists()) {
                        localFile.getParent().createFolder();
                    }

                    localFile.copyFrom(f, new AllFileSelector());
                }
            }
        } // for (FileObject f : children)
    } // searchCurrentDirectory(String path, FileObject file)

} // class App


=== end App.java ===

---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@commons.apache.org
For additional commands, e-mail: user-help@commons.apache.org


Mime
View raw message