gump-general mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anto...@apache.org
Subject cvs commit: gump/python/gump/test sync.py
Date Tue, 09 Mar 2004 14:14:33 GMT
antoine     2004/03/09 06:14:33

  Modified:    python/gump/utils sync.py
               python/gump/test sync.py
  Log:
  Make Sync extend Annotable
  also add support for copy
  
  Revision  Changes    Path
  1.2       +122 -88   gump/python/gump/utils/sync.py
  
  Index: sync.py
  ===================================================================
  RCS file: /home/cvs/gump/python/gump/utils/sync.py,v
  retrieving revision 1.1
  retrieving revision 1.2
  diff -u -r1.1 -r1.2
  --- sync.py	4 Mar 2004 21:38:47 -0000	1.1
  +++ sync.py	9 Mar 2004 14:14:32 -0000	1.2
  @@ -70,19 +70,28 @@
   from gump.utils.work import *
   from gump.utils.file import *
   from gump.utils.launcher import *
  -class Sync:
  +from gump.utils.note import *
  +class Sync(Annotatable):
       """
       this class can be used to sync two directories
       x = Sync(sourcedir, targetdir) # construct
       x.execute() #run
       if targetdir does not exist, it will be created
       if sourcedir does not exist, the class will raise an IOError
  +    the class can be also used to copy from one directory to another
  +    x = Sync(sourcedir, targetdir, 1)
       """
  -    def __init__(self, sourcedir, targetdir):
  +    def __init__(self, sourcedir, targetdir, copyflag = 0):
  +        Annotatable.__init__(self)
           self.sourcedir = sourcedir
           self.targetdir = targetdir
  +        self.copyflag = copyflag
       def execute(self):
  -        log.info('Starting sync from [' + self.sourcedir + ']')
  +        if self.copyflag:
  +            action = 'copy'
  +        else:
  +            action = 'sync'
  +        log.info('Starting %s  from [%s]' % (action,self.sourcedir))
           log.info('        target dir [' + self.targetdir + ']')
           if not os.path.exists(self.sourcedir):
               log.error('Exiting sync, source directory does not exist [' + self.sourcedir
+ ']')
  @@ -97,92 +106,117 @@
                   os.makedirs(self.targetdir)
               except Exception, details:
                   log.exception('failed on ' + str(details))
  -                return
  -        copytree(self.sourcedir, self.targetdir, 1)    
  +                raise IOError, 'could not create directory [' + self.targetdir + ']'
  +        self.copytree(self.sourcedir, self.targetdir, 1)    
               
  -def copytree(src, dst, symlinks=0):
  -    names = os.listdir(src)
  -    try:
  -        result = os.stat(dst)
  -    except Exception, details:
  -        result = None
  -    # handle case where result exists but is not a directory    
  -    if result and not S_ISDIR(result[ST_MODE]):
  -        remove(dst)
  -        result = None
  -    if not result:    
  -        os.makedirs(dst)
  -    if result:
  -        names2 = os.listdir(dst)
  -        epurate(src, dst, names, names2)    
  -    for name in names:
  -        srcname = os.path.join(src, name)
  -        dstname = os.path.join(dst, name)
  -        try:
  -            if symlinks and os.path.islink(srcname):
  -                linkto = os.readlink(srcname)
  -                os.symlink(linkto, dstname)
  -            elif os.path.isdir(srcname):
  -                copytree(srcname, dstname, symlinks)
  -            else:
  -                maybecopy(srcname, dstname)
  -        except (IOError, os.error), why:
  -            message = "Can't copy [%s] to [%s]: [%s]" % (`srcname`, `dstname`, str(why))
  -            log.exception(message)
  -            raise IOError, message
  -def epurate(sourcedir, destdir, acceptablefiles, existingfiles):
  -    """
  -    this routine will delete from a set of existing files
  -    in a directory the one which are not part of an 
  -    array of acceptablefiles
  -    somedir = directory where the epuration is to take place                          
           
  -    """
  -    for afile in existingfiles:
  -        fullsourcefile = os.path.join(sourcedir, afile)
  -        fulldestfile = os.path.join(destdir, afile)
  -        if not afile in acceptablefiles:
  -            tobedeleted = os.path.join(destdir, afile)
  -            result = os.stat(tobedeleted)
  -            if S_ISDIR(result[ST_MODE]):
  -                log.debug('attempting to remove directory [%s]' % (`tobedeleted`))
  -                shutil.rmtree(tobedeleted)
  -            else:    
  -                log.debug('attempting to remove file [%s]' % (`tobedeleted`))
  -                os.remove(tobedeleted)
  -        elif os.path.isdir(fullsourcefile) and not os.path.isdir(fulldestfile):
  -            log.debug('removing file [%s] to be replaced by directory' 
  -            %(`fulldestfile`))
  -            os.remove(fulldestfile)
  -        elif os.path.isfile(fullsourcefile) and os.path.isdir(fulldestfile):          
   
  -            log.debug('removing directory [%s] to be replaced by file' 
  -            %(`fulldestfile`))
  -            shutil.rmtree(fulldestfile)
  -            
  -def maybecopy(srcname, dstname):
  -    """
  -    copy a file from srcname to dstname if 
  -    dstname does not exist
  -    or srcname and dstname have different dates
  -    or srcname and dstname have different sizes
  -    """
  -    result = os.stat(srcname)
  -    try:
  -        result2 = os.stat(dstname)
  -    except (Exception), details:
  -        result2 = None
  -    okcopy = 0
  -    if not result2:
  -        okcopy = 1
  -    elif S_ISDIR(result2[ST_MODE]):
  -        shutil.rmtree(dstname)
  -        okcopy = 1
  -    elif result[ST_SIZE] != result2[ST_SIZE]:
  -        okcopy = 1
  -    elif result[ST_MTIME] != result2[ST_MTIME]:
  -        okcopy = 1
  -    if okcopy:
  -        log.debug("Attempting copy from [%s] to [%s]" %(`srcname`, `dstname`))    
  -        shutil.copy2(srcname, dstname)
  +    def copytree(self, src, dst, symlinks=0):
  +        names = os.listdir(src)
  +        try:
  +            result = os.stat(dst)
  +        except Exception, details:
  +            result = None
  +        # handle case where result exists but is not a directory    
  +        if result and not S_ISDIR(result[ST_MODE]):
  +            remove(dst)
  +            result = None
  +        if not result:    
  +            os.makedirs(dst)
  +        if result:
  +            names2 = os.listdir(dst)
  +            self.removenonmatching(src, dst, names, names2)
  +            if not self.copyflag:    
  +                self.epurate(src, dst, names, names2)    
  +        for name in names:
  +            srcname = os.path.join(src, name)
  +            dstname = os.path.join(dst, name)
  +            try:
  +                if symlinks and os.path.islink(srcname):
  +                    linkto = os.readlink(srcname)
  +                    os.symlink(linkto, dstname)
  +                elif os.path.isdir(srcname):
  +                    self.copytree(srcname, dstname, symlinks)
  +                else:
  +                    self.maybecopy(srcname, dstname)
  +            except (IOError, os.error), why:
  +                message = "Can't copy [%s] to [%s]: [%s]" % (`srcname`, `dstname`, str(why))
  +                log.exception(message)
  +                raise IOError, message
  +    def epurate(self, sourcedir, destdir, acceptablefiles, existingfiles):        
  +        """
  +        this routine will delete from a set of existing files
  +        in a directory the one which are not part of an 
  +        array of acceptablefiles
  +        sourcedir = directory from which the copy is taking place
  +        destdir = directory where the epuration is to take place 
  +        acceptablefiles = array of filenames of files which are accepted at destination
  +        existingfiles = array of filenames which exist at destination                 
                   
  +        """
  +        for afile in existingfiles:
  +            fullsourcefile = os.path.join(sourcedir, afile)
  +            fulldestfile = os.path.join(destdir, afile)
  +            if not afile in acceptablefiles:
  +                tobedeleted = os.path.join(destdir, afile)
  +                result = os.stat(tobedeleted)
  +                if S_ISDIR(result[ST_MODE]):
  +                    log.debug('attempting to remove directory [%s]' % (`tobedeleted`))
  +                    shutil.rmtree(tobedeleted)
  +                else:    
  +                    log.debug('attempting to remove file [%s]' % (`tobedeleted`))
  +                    os.remove(tobedeleted)
  +    def removenonmatching(self, sourcedir, destdir, acceptablefiles, existingfiles):
  +        """
  +        this routine will remove from destination
  +        the entries which are files at destination and directory at source
  +        the entries which are directory at destination and files at source
  +        leaves untouched entries which exist both at source and at destination
  +        sourcedir = directory from which the copy is taking place
  +        destdir = directory where the epuration is to take place 
  +        acceptablefiles = array of filenames of files which are accepted at destination
  +        existingfiles = array of filenames which exist at destination                 
                   
  +        """
  +        removed = []
  +        for afile in existingfiles:
  +            fullsourcefile = os.path.join(sourcedir, afile)
  +            fulldestfile = os.path.join(destdir, afile)
  +            if afile in acceptablefiles:
  +                if os.path.isdir(fullsourcefile) and not os.path.isdir(fulldestfile):
  +                    log.debug('removing file [%s] to be replaced by directory' 
  +                    %(`fulldestfile`))
  +                    os.remove(fulldestfile)
  +                    removed.append(afile)
  +                elif os.path.isfile(fullsourcefile) and os.path.isdir(fulldestfile):  
           
  +                    log.debug('removing directory [%s] to be replaced by file' 
  +                    %(`fulldestfile`))
  +                    shutil.rmtree(fulldestfile)
  +                    removed.append(afile)
  +        for afile in removed:
  +            existingfiles.remove(afile)             
  +                
  +    def maybecopy(self, srcname, dstname):
  +        """
  +        copy a file from srcname to dstname if 
  +        dstname does not exist
  +        or srcname and dstname have different dates
  +        or srcname and dstname have different sizes
  +        """
  +        result = os.stat(srcname)
  +        try:
  +            result2 = os.stat(dstname)
  +        except (Exception), details:
  +            result2 = None
  +        okcopy = 0
  +        if not result2:
  +            okcopy = 1
  +        elif S_ISDIR(result2[ST_MODE]):
  +            shutil.rmtree(dstname)
  +            okcopy = 1
  +        elif result[ST_SIZE] != result2[ST_SIZE]:
  +            okcopy = 1
  +        elif result[ST_MTIME] != result2[ST_MTIME]:
  +            okcopy = 1
  +        if okcopy:
  +            log.debug("Attempting copy from [%s] to [%s]" %(`srcname`, `dstname`))    
  +            shutil.copy2(srcname, dstname)
           
               
               
  
  
  
  1.2       +65 -17    gump/python/gump/test/sync.py
  
  Index: sync.py
  ===================================================================
  RCS file: /home/cvs/gump/python/gump/test/sync.py,v
  retrieving revision 1.1
  retrieving revision 1.2
  diff -u -r1.1 -r1.2
  --- sync.py	4 Mar 2004 21:38:47 -0000	1.1
  +++ sync.py	9 Mar 2004 14:14:33 -0000	1.2
  @@ -116,21 +116,21 @@
           try:
               result = os.stat(self.destination)
           except Exception, details:
  -            raiseIssue(['destination directory was not created', self.destination])
  +            self.raiseIssue(['destination directory was not created', self.destination])
           try:
               result = os.stat(self.destination_subdir1)
           except Exception, details:
  -            raiseIssue(['destination_subdir1 directory was not created', self.destination_subdir1])
  +            self.raiseIssue(['destination_subdir1 directory was not created', self.destination_subdir1])
           result_source = None
           result_destination = None    
           try:
               result_source = os.stat(self.source_alphatxt)    
           except Exception, details:
  -            raiseIssue(['file was not created', self.source_alphatxt])
  +            self.raiseIssue(['file was not created', self.source_alphatxt])
           try:
               result_destination = os.stat(self.destination_alphatxt)
           except Exception, details:
  -            raiseIssue(['file was not created', self.destination_alphatxt])
  +            self.raiseIssue(['file was not created', self.destination_alphatxt])
           log.debug("size of file [%s] is %i" % (`self.destination_alphatxt`,
           result_destination[stat.ST_SIZE]))    
           log.debug("modification date of file [%s] is %s" % 
  @@ -166,17 +166,17 @@
           os.utime(self.destination_alphatxt, (epoch_sometime, epoch_sometime))
           mySync.execute()
           if os.path.exists(destination_junktxt):
  -            raiseIssue(['junk file was not deleted', destination_junktxt])
  +            self.raiseIssue(['junk file was not deleted', destination_junktxt])
           result_source = None
           result_destination = None    
           try:
               result_source = os.stat(self.source_alphatxt)    
           except Exception, details:
  -            raiseIssue(['file was not created', self.source_alphatxt])
  +            self.raiseIssue(['file was not created', self.source_alphatxt])
           try:
               result_destination = os.stat(self.destination_alphatxt)
           except Exception, details:
  -            raiseIssue(['file was not created', self.destination_alphatxt])
  +            self.raiseIssue(['file was not created', self.destination_alphatxt])
           log.debug("size of file [%s] is %i" % (`self.destination_alphatxt`,
           result_destination[stat.ST_SIZE]))    
           log.debug("modification date of file [%s] is %s" % 
  @@ -208,18 +208,18 @@
           shutil.copy2(self.source_alphatxt, junk_file2)
           mySync.execute()
           if os.path.isdir(self.destination_alphatxt):
  -            raiseIssue(['destination text file remained a directory',
  +            self.raiseIssue(['destination text file remained a directory',
                self.destination_alphatxt])
           result_source = None
           result_destination = None    
           try:
               result_source = os.stat(self.source_alphatxt)    
           except Exception, details:
  -            raiseIssue(['file was not created', self.source_alphatxt])
  +            self.raiseIssue(['file was not created', self.source_alphatxt])
           try:
               result_destination = os.stat(self.destination_alphatxt)
           except Exception, details:
  -            raiseIssue(['file was not created', self.destination_alphatxt])
  +            self.raiseIssue(['file was not created', self.destination_alphatxt])
           log.debug("size of file [%s] is %i" % (`self.destination_alphatxt`,
           result_destination[stat.ST_SIZE]))    
           log.debug("modification date of file [%s] is %s" % 
  @@ -253,18 +253,66 @@
           shutil.copy2(junk_source_file1, junk_source_file2)
           mySync.execute()
           if os.path.isfile(self.destination_alphatxt):
  -            raiseIssue(['destination text file remained a file',
  +            self.raiseIssue(['destination text file remained a file',
                self.destination_alphatxt])
           self.genericCompare(self.source_alphatxt, self.destination_alphatxt)
  -        log.debug('finished')     
  +    def testSymbolicLink(self):
  +        """
  +        this test only runs on operating systems where os.name will return
  +        posix
  +        the setUp gets done
  +        a symbolic link sl pointing to source_subdir1 gets created
  +        the sync gets done
  +        we want to check whether sl exists on the destination side
  +        as a symbolic link
  +        """
  +        if os.name == 'posix': 
  +            dstname = os.path.join(self.source_subdir1, 'myfirstlink')
  +            os.symlink('subdir1', dstname)    
  +            mySync = Sync(self.source, self.destination)
  +            mySync.execute()
  +            self.genericCompare(self.source, self.destination)
  +    def testCopy1(self):
  +        """
  +        the setUp gets done
  +        a sync runs
  +        then source_alphatxt is deleted
  +        another file source_betatxt is created
  +        a sync with the copy flag runs
  +        the test will check that :
  +            alphatxt remains on the destination side
  +            betatxt gets copied
  +        """    
  +        mySync = Sync(self.source, self.destination)
  +        mySync.execute()
  +        os.remove(self.source_alphatxt)
  +        betatxt = "beta.txt"
  +        source_betatxt = os.path.join(self.source_subdir1, betatxt)
  +        destination_betatxt = os.path.join(self.destination_subdir1, betatxt)
  +        myfile = file(source_betatxt, 'w+')
  +        myfile.write('Hello World')
  +        myfile.close()
  +        # Sat, 20 May 2000 12:07:40 +0000
  +        sometime=[2000,5,20,12,7,40,5,141,-1]
  +        epoch_sometime = time.mktime(sometime)
  +        os.utime(source_betatxt, (epoch_sometime, epoch_sometime))
  +        mySync = Sync(self.source, self.destination, 1)
  +        mySync.execute()
  +        # check that alpha.txt was preserved on the destination side
  +        if not os.path.exists(self.destination_alphatxt):
  +            self.raiseIssue(['file was deleted on the destination side in a copy'
  +            + 'operation', self.source_alphatxt])
  +        # check that beta.txt was copied    
  +        self.compareFiles(source_betatxt, destination_betatxt)    
  +             
       def genericCompare(self, mysource, mydestination):
           """
           compare 2 directories source and destination
           """ 
           if not os.path.isdir(mysource):
  -            raiseIssue([mysource, ' not a directory'])
  +            self.raiseIssue([mysource, ' not a directory'])
           if not os.path.isdir(mydestination):
  -            raiseIssue([mydestination, ' not a directory'])
  +            self.raiseIssue([mydestination, ' not a directory'])
           names = os.listdir(mysource)
           for aname in names:
               inode_source = os.path.join(mysource, aname)
  @@ -273,7 +321,7 @@
                   self.compareFiles(inode_source, inode_dest)
               elif os.path.islink(inode_source):
                   if not os.path.islink(inode_dest):
  -                    raiseIssue([inode_dest, ' not a symbolic link'])
  +                    self.raiseIssue([inode_dest, ' not a symbolic link'])
                   linkto_source = os.readlink(inode_source)
                   linkto_dest = os.readlink(inode_dest)
                   self.assertTrue([inode_dest, ' points to ', linkto_source ],
  @@ -291,11 +339,11 @@
           try:
               result_source = os.stat(inode_source)
           except Exception, details:
  -            raiseIssue(['could not stat ', inode_source])
  +            self.raiseIssue(['could not stat ', inode_source])
           try:
               result_dest = os.stat(inode_dest)
           except Exception, details:
  -            raiseIssue(['could not stat ', inode_dest])
  +            self.raiseIssue(['could not stat ', inode_dest])
           self.assertTrue("modtime is equal for [%s] compared to [%s]"
           %(`inode_source`,`inode_dest`),
           result_source[stat.ST_MTIME]==result_dest[stat.ST_MTIME])
  
  
  

---------------------------------------------------------------------
To unsubscribe, e-mail: general-unsubscribe@gump.apache.org
For additional commands, e-mail: general-help@gump.apache.org


Mime
View raw message