gump-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From leosim...@apache.org
Subject svn commit: r124502 - in gump/branches/Dec04MajorCleanup: . pygump pygump/python/gump pygump/python/gump/engine pygump/python/gump/plugins
Date Fri, 07 Jan 2005 14:41:30 GMT
Author: leosimons
Date: Fri Jan  7 06:41:28 2005
New Revision: 124502

URL: http://svn.apache.org/viewcvs?view=rev&rev=124502
Log:
And then the engine was done!

* a lot of bugfixes in the modeller
* a new Worker that will take care of walking the graph
* rearrange stuff to make it easy to change different things, see what is easy to change, etc
* some documentation
* some more todo flags of course...

Added:
   gump/branches/Dec04MajorCleanup/pygump/python/gump/config.py
   gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/walker.py
Modified:
   gump/branches/Dec04MajorCleanup/gump
   gump/branches/Dec04MajorCleanup/pygump/main.py
   gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py
   gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py
   gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/__init__.py
   gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/dynagumper.py

Modified: gump/branches/Dec04MajorCleanup/gump
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/gump?view=diff&rev=124502&p1=gump/branches/Dec04MajorCleanup/gump&r1=124501&p2=gump/branches/Dec04MajorCleanup/gump&r2=124502
==============================================================================
--- gump/branches/Dec04MajorCleanup/gump	(original)
+++ gump/branches/Dec04MajorCleanup/gump	Fri Jan  7 06:41:28 2005
@@ -86,6 +86,7 @@
       argument is the "project expression" detailing what to run. You
       will usually want "all" here.
 "
+# TODO: document Pygump CLI args!
       ;;
     kill)
       echo "

Modified: gump/branches/Dec04MajorCleanup/pygump/main.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/main.py?view=diff&rev=124502&p1=gump/branches/Dec04MajorCleanup/pygump/main.py&r1=124501&p2=gump/branches/Dec04MajorCleanup/pygump/main.py&r2=124502
==============================================================================
--- gump/branches/Dec04MajorCleanup/pygump/main.py	(original)
+++ gump/branches/Dec04MajorCleanup/pygump/main.py	Fri Jan  7 06:41:28 2005
@@ -49,6 +49,54 @@
 CRITICAL = 1
 
 
+def get_parser(_homedir, _hostname, _projects, _workdir, _logdir, _workspace):
+    """Pygump uses the optparse package to provide the CLI.
+    
+    To add new options to pygump, change this method and document the changes
+    in the main 'gump' shell script, then change gump.config to interpret
+    those values and do something useful with them.
+    
+    Please be aware that other parts of main.py and of gump.config do set and
+    modify some of the results the option parser gives, so if somethin does
+    not work as expected, you may need to modify other parts of those files.
+    
+    The provided arguments are defaults retrieved from environment variables
+    and the like.
+    """
+    # TODO: make sure no CLI settings are overridden!
+    from optparse import OptionParser
+    parser = OptionParser()
+    parser.add_option("--debug",
+                      action="store_true",
+                      default=False)
+    parser.add_option("--homedir",
+                      action="store",
+                      default=_homedir)
+    parser.add_option("--hostname",
+                      action="store",
+                      default=_hostname)
+    parser.add_option("-p",
+                      "--project",
+                      action="append",
+                      dest="projects",
+                      default=_projects)
+    parser.add_option("--workdir",
+                      action="store",
+                      default=_workdir)
+    parser.add_option("--logdir",
+                      action="store",
+                      default=_logdir)
+    parser.add_option("-w",
+                      "--workspace",
+                      action="store",
+                      default=_workspace)
+    parser.add_option("--no-updates",
+                      action="store_true",
+                      dest="no_updates",
+                      default=False)
+    return parser
+
+
 class Error(Exception):
     """Generic error thrown for all internal pygump main module exceptions."""
     pass
@@ -153,6 +201,7 @@
     put into the options instance. It doesn't merge in the profile or anything
     like that, that is left up to the engine.
     """
+    # TODO: remove core options from the workspace completely!
     domtree             = minidom.parse(filename)
     w                   = domtree.getElementsByTagName('workspace').item(0)
     options.name        = w.getAttribute('name')
@@ -327,36 +376,7 @@
     _workspace     = os.path.join(_homedir, "metadata", "%s.xml" % (_hostname))
     
     # get basic settings from commandline arguments
-    from optparse import OptionParser
-    parser = OptionParser()
-    parser.add_option("--debug",
-                      action="store_true",
-                      default=False)
-    parser.add_option("--homedir",
-                      action="store",
-                      default=_homedir)
-    parser.add_option("--hostname",
-                      action="store",
-                      default=_hostname)
-    parser.add_option("-p",
-                      "--project",
-                      action="append",
-                      dest="projects",
-                      default=_projects)
-    parser.add_option("--workdir",
-                      action="store",
-                      default=_workdir)
-    parser.add_option("--logdir",
-                      action="store",
-                      default=_logdir)
-    parser.add_option("-w",
-                      "--workspace",
-                      action="store",
-                      default=_workspace)
-    parser.add_option("--no-updates",
-                      action="store_true",
-                      dest="no_updates",
-                      default=False)
+    parser = get_parser(_homedir, _hostname, _projects, _workdir, _logdir, _workspace)
     options, args = parser.parse_args()
     
     options.starttime = time.strftime('%d %b %Y %H:%M:%S', time.localtime())

Added: gump/branches/Dec04MajorCleanup/pygump/python/gump/config.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/config.py?view=auto&rev=124502
==============================================================================
--- (empty file)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/config.py	Fri Jan  7 06:41:28 2005
@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module provides the "component wiring" for the gump application."""
+
+__copyright__ = "Copyright (c) 2005 The Apache Software Foundation"
+__license__   = "http://www.apache.org/licenses/LICENSE-2.0"
+
+import os
+import logging
+
+def get_config(settings):
+    """Convert the settings object into something more specific.
+
+    The 'main' module uses the 'optparse' module to generate a Values instance,
+    which is the argument provided to this method. The reason we put this
+    function and the Config class in between those settings and the main
+    engine is that we can change gump internals more easily while keeping the
+    CLI interface the same, with the integration point being isolated to this
+    config module.
+    
+    Nonetheless, if possible, it is preferable to set options through CLI
+    rather than in this method. Modify the get_parser() method in the main
+    module to support that.
+    """
+    config = Config(settings)
+    
+    if settings.debug:
+        config.log_level       = logging.DEBUG
+    else:
+        config.log_level       = logging.INFO
+
+    # TODO: change main.py to do it like this
+    config.paths_home      = settings.homedir
+    config.paths_work      = settings.workdir
+    config.paths_logs      = settings.logdir
+    config.paths_workspace = settings.workspace
+    config.do_update       = not settings.no_updates
+    config.start_time      = settings.starttimeutc
+    
+    config.projects        = settings.projects
+    
+    config.mail_server     = settings.mailserver
+    config.mail_server_port = settings.mailport
+    config.mail_to         = settings.mailto
+    config.mail_from       = settings.mailfrom
+    
+    # TODO: set defaults in main.py instead
+    config.database_server = "localhost"
+    if hasattr(settings,"databaseserver"): config.database_server = settings.databaseserver
+    config.database_port   = 3306
+    if hasattr(settings,"databaseport"): config.database_port = settings.databaseport
+    config.database_name   = "gump"
+    if hasattr(settings,"databasename"): config.database_name = settings.databasename
+    config.database_user   = "gump"
+    if hasattr(settings,"databaseuser"): config.database_user = settings.databaseuser
+    config.database_password   = "gump"
+    if hasattr(settings,"databasepassword"): config.database_password = settings.databasepassword
+    
+    return config
+
+
+def get_plugins(config):
+    """This method returns an ordered list of plugins.
+    
+    These plugins work together to form "the meat" of the gump application.
+    Changing what this method returns allows changing nearly everything but
+    the core behaviour of the gump engine.
+    
+    The config argument provided is an instance of the _Config class that is
+    returned from the get_config method below.
+    """
+    plugins = []
+
+    from gump.plugins import LoggingPlugin
+    log = get_logger(config.log_level, "plugin-log")
+    plugins.append(LoggingPlugin(log))
+    
+    from gump.plugins.dynagumper import Dynagumper
+    db = get_db(config)
+    log = get_logger(config.log_level, "plugin-dynagumper")
+    plugins.append(Dynagumper(db, log))
+
+    # TODO: append more plugins here...
+    
+    return plugins
+
+
+def get_error_handler(config):
+    """This method returns the error handler that deals with naughty plugins.
+    
+    Though every plugin is free to do proper error handling and recovery, the
+    ones that do usually should not bring gump to a full stop. That is why a
+    central error handler is defined that catches all exceptions thrown by
+    all plugins and figures out what to do with it. This function provides
+    that error handler.
+
+    The config argument provided is an instance of the _Config class that is
+    returned from the get_config method below.
+    """
+
+    # TODO: implement an error handler that does actual recovery...
+    
+    from gump.plugins import LoggingErrorHandler
+    log = get_logger(config.log_level, "plugin-error-handler")
+    return LoggingErrorHandler(log)
+
+###
+### Changing anything below this line is for advanced gump hackers only!
+###
+
+class Config:
+    """Central class for configuring the gump engine.
+
+    This class is mostly an empty "container" on which the get_config method
+    above sets properties. The only bit of intelligence in this class is
+    within the defined __getattr__ method that computes config values based
+    on other config values.
+    """
+    def __init__(self, cli_provided_settings):
+        self.settings = cli_provided_settings
+
+    def __getattr__(self,name):
+        """Calculate missing settings from other settings at runtime."""
+        if hasattr(self.settings, name):
+            return self.settings.name
+
+        if name == 'debug':
+            return self.loglevel >= logging.DEBUG
+        if name == 'paths_pygump':
+            return os.path.jion(self.paths_home, "pygump")
+        if name == 'paths_metadata':
+            return os.path.join(self.paths_home, "metadata")
+        if name == 'do_mail':
+            return self.mail_server and self.mail_server_port and self.mail_to and self.mail_from
+        
+        # unknown, raise error
+        raise AttributeError, name
+
+
+def get_logger(level, name):
+    """Provide a logging implementation for the given level and name."""
+    logging.basicConfig()
+    log = logging.getLogger(name)
+    log.setLevel(level)
+    return log
+
+
+def get_db(config):
+    """Provide a database implementation."""
+    from gump.util.mysql import Database
+    db = Database(config) #TODO!
+    return db
+
+
+def get_vfs(filesystem_root, cache_dir):
+    """Provide a VFS implementation."""
+    from gump.util.io import VFS
+    return VFS(filesystem_root, cache_dir)
+
+
+def get_modeller_loader(log, vfs=None, mergefile=None, dropfile=None):
+    """Provide a Loader implementation."""
+    from gump.engine.modeller import Loader
+    return Loader(log, vfs, mergefile, dropfile)
+
+
+def get_modeller_normalizer(log):
+    """Provide a Normalizer implementation."""
+    from gump.engine.modeller import Normalizer
+    return Normalizer(log)
+
+
+def get_modeller_objectifier(log):
+    """Provide a Objectifier implementation."""
+    from gump.engine.modeller import Objectifier
+    return Objectifier(log)
+
+
+def get_modeller_verifier():
+    """Provide a Verifier implementation."""
+    from gump.engine.modeller import Verifier
+    return Verifier()
+
+
+def get_walker():
+    """Provide a Walker implementation."""
+    from gump.engine.walker import Walker
+    return Walker()
+
+
+def get_plugin(config):
+    """Provide a Plugin implementation."""
+    from gump.plugins import MulticastPlugin
+    
+    plugins = get_plugins(config)
+    error_handler = get_error_handler(config)
+    
+    return MulticastPlugin(plugins, error_handler)

Modified: gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py?view=diff&rev=124502&p1=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py&r1=124501&p2=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py&r2=124502
==============================================================================
--- gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py	(original)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py	Fri Jan  7 06:41:28 2005
@@ -29,21 +29,21 @@
 on are passed into the constructor when first creating the instance. This is
 all handled by the main() function. Doing things this way greatly decreases
 the amount of coupling between the different modules. For example, the
-gump.engine module is the only module that actually has an "import logging".
+gump.config module is the only module that actually has an "import logging".
 It creates all the log instances to pass into its helpers. This means that
 completely rewiring or disabling logging is a simple as modifying the
-_get_logger() function in this module.
+get_logger() function in the gump.config module.
 """
 
 __copyright__ = "Copyright (c) 2004-2005 The Apache Software Foundation"
 __license__   = "http://www.apache.org/licenses/LICENSE-2.0"
 
-import os
 import logging
 
 from xml import dom
 from xml.dom import minidom
 
+from gump.config import *
 from gump.util.io import open_file_or_stream
 
 def main(settings):
@@ -64,29 +64,31 @@
     _banner(settings.version)
     
     # get engine config
-    config = _get_config(settings)
+    config = get_config(settings)
     
     # get engine dependencies
-    log = _get_logger(config.log_level, "engine")
-    db  = _get_db(config)
+    log = get_logger(config.log_level, "engine")
     
     vfsdir = os.path.join(config.paths_work, "vfs-cache")
     if not os.path.isdir(vfsdir):
         os.mkdir(vfsdir);
-    vfs = _get_vfs(config.paths_metadata, vfsdir)
+    vfs = get_vfs(config.paths_metadata, vfsdir)
     
-    modeller_log = _get_logger(config.log_level, "modeller")
-    modeller_loader = _get_modeller_loader(modeller_log, vfs)
-    modeller_normalizer = _get_modeller_normalizer()
-    modeller_objectifier = _get_modeller_objectifier(modeller_log)
-    modeller_verifier = _get_modeller_verifier()
+    modeller_log = get_logger(config.log_level, "modeller")
+    modeller_loader = get_modeller_loader(modeller_log, vfs)
+    modeller_normalizer = get_modeller_normalizer(modeller_log)
+    modeller_objectifier = get_modeller_objectifier(modeller_log)
+    modeller_verifier = get_modeller_verifier()
     
     mergefile = os.path.join(config.paths_work, "merge.xml")
     dropfile = os.path.join(config.paths_work, "dropped.xml")
-
+    
+    walker = get_walker()
+    visitor = get_plugin(config)
+    
     # create engine
-    engine = _Engine(log, db, modeller_loader, modeller_normalizer,
-                     modeller_objectifier, modeller_verifier,
+    engine = _Engine(log, modeller_loader, modeller_normalizer,
+                     modeller_objectifier, modeller_verifier, walker, visitor,
                      config.paths_workspace, mergefile, dropfile)
     
     # run it
@@ -109,120 +111,17 @@
 ### FACTORY METHODS
 ###
 
-def _get_config(settings):
-    """Convert the settings object into something more specific.
-
-    The reason we put this function and the Config
-    class in between is that we can change gump internals while keeping the CLI
-    interface the same more easily, with the integration point being isolated to
-    this method and the Config class definition.
-    """
-    config = _Config()
-    
-    if settings.debug:
-        config.log_level       = logging.DEBUG
-    else:
-        config.log_level       = logging.INFO
-
-    config.hostname        = settings.hostname
-    config.projects        = settings.projects
-    config.paths_home      = settings.homedir
-    config.paths_work      = settings.workdir
-    config.paths_logs      = settings.logdir
-    config.paths_workspace = settings.workspace
-    config.do_update       = not settings.no_updates
-    config.start_time      = settings.starttimeutc
-    
-    config.projects        = settings.projects
-    
-    config.mail_server     = settings.mailserver
-    config.mail_server_port = settings.mailport
-    config.mail_to         = settings.mailto
-    config.mail_from       = settings.mailfrom
-    
-    # TODO: set defaults in main.py instead
-    config.database_server = "localhost"
-    if hasattr(settings,"databaseserver"): config.database_server = settings.databaseserver
-    config.database_port   = 3306
-    if hasattr(settings,"databaseport"): config.database_port = settings.databaseport
-    config.database_name   = "gump"
-    if hasattr(settings,"databasename"): config.database_name = settings.databasename
-    config.database_user   = "gump"
-    if hasattr(settings,"databaseuser"): config.database_user = settings.databaseuser
-    config.database_password   = "gump"
-    if hasattr(settings,"databasepassword"): config.database_password = settings.databasepassword
-    
-    return config
-
-def _get_logger(level, name):
-    """Provide a logging implementation for the given level and name."""
-    logging.basicConfig()
-    log = logging.getLogger(name)
-    log.setLevel(level)
-    return log
-
-def _get_db(config):
-    """Provide a database implementation."""
-    from gump.util.mysql import Database
-    db = Database(config) #TODO!
-    return db
-
-def _get_vfs(filesystem_root, cache_dir):
-    """Provide a VFS implementation."""
-    from gump.util.io import VFS
-    return VFS(filesystem_root, cache_dir)
-
-def _get_modeller_loader(log, vfs=None, mergefile=None, dropfile=None):
-    """Provide a Loader implementation."""
-    from gump.engine.modeller import Loader
-    return Loader(log, vfs, mergefile, dropfile)
-
-def _get_modeller_normalizer():
-    """Provide a Normalizer implementation."""
-    from gump.engine.modeller import Normalizer
-    return Normalizer()
-
-def _get_modeller_objectifier(log):
-    """Provide a Objectifier implementation."""
-    from gump.engine.modeller import Objectifier
-    return Objectifier(log)
-
-def _get_modeller_verifier():
-    """Provide a Verifier implementation."""
-    from gump.engine.modeller import Verifier
-    return Verifier()
-
-###
-### Classes
-###
-
-class _Config:
-    def __getattr__(self,name):
-        """Calculate missing settings from other settings at runtime."""
-        if name == 'debug':
-            return self.loglevel >= logging.DEBUG
-        if name == 'paths_pygump':
-            return os.path.jion(self.paths_home, "pygump")
-        if name == 'paths_metadata':
-            return os.path.join(self.paths_home, "metadata")
-        if name == 'do_mail':
-            return self.mail_server and self.mail_server_port and self.mail_to and self.mail_from
-        
-        # unknown, raise error
-        raise AttributeError, name
-
 class _Engine:
     """This is the core of the core of the pygump application."""
     
-    def __init__(self, log, db, workspace_loader, workspace_normalizer,
-                 workspace_objectifier, workspace_verifier,
+    def __init__(self, log, workspace_loader, workspace_normalizer,
+                 workspace_objectifier, workspace_verifier, walker, visitor,
                  workspace, merge_to=None, drop_to=None):
         """Store all config and dependencies as properties.
         
         Arguments:
             
             - log -- the log to write debug and error messages to.
-            - db -- the database to store all activity in.
             - workspace_loader -- the component providing the dom tree.
             - workspace_normalizer -- the component transforming the dom tree
                 into a standard format
@@ -230,17 +129,22 @@
                 object form
             - workspace_verifier -- the component making sure the object model
                 is correct
+            - walker -- the component that knows how to traverse the gump
+                model in dependency order
+            - visitor -- the component that gets called by the walker while
+                visiting parts of the model
 
             - workspace -- the resource containing the workspace xml.
             - merge_to -- the resource to write the merged workspace xml to.
             - drop_to -- the resource to write the dropped projects xml to.
         """
         self.log = log
-        self.db = db
         self.workspace_loader = workspace_loader
         self.workspace_normalizer = workspace_normalizer
         self.workspace_objectifier = workspace_objectifier
         self.workspace_verifier = workspace_verifier
+        self.walker = walker
+        self.visitor = visitor
 
         self.workspace = open_file_or_stream(workspace,'r')
         self.merge_to = open_file_or_stream(merge_to,'w')
@@ -251,32 +155,29 @@
         try:
             # * merge workspace into big DOM tree
             (domtree, dropped_nodes) = self.workspace_loader.get_workspace_tree(self.workspace)
+            
             # * clean it up and structure it properly
-            self.workspace_normalizer.normalize(domtree)
+            domtree = self.workspace_normalizer.normalize(domtree)
+            
             # * write the merged, normalized tree out to a new xml file
             self._write_merge_files(domtree, dropped_nodes)
+            
             # * convert that DOM tree into python objects
             workspace = self.workspace_objectifier.get_workspace(domtree)
+            
             # * we're done with the xml stuff, allow GC
             domtree.unlink()
             for node in dropped_nodes:
                 node.unlink()
+                
             # * verify that our model is correct (for example, that it has
             #   no circular dependencies)
             self.workspace_verifier.verify(domtree)
-            # * store those objects in the database
-
-            raise RuntimeError, "Not Implemented!"
             
-            self.store_workspace(self.workspace) #TODO
-
-            # * determine the tasks to perform
-            self.tasks = self.create_ordered_tasklist() #TODO
+            # * Pfew! All done. Now actually start *doing* stuff.
+            self.walker.walk(workspace, self.visitor)
             
-            # * now make the workers perform those tasks
-            self.create_workers() #TODO
-            self.start_workers() #TODO
-            self.wait_for_workers() #TODO
+            # That's it? Yeah! All other functionality is in the visitors :-D
         except:
             self.log.exception("Fatal error during run!")
     
@@ -287,7 +188,7 @@
         dropped because of a HREF resolution issue.
         """
         if self.merge_to:
-            self.merge_to.write( domtree.toprettyxml() )
+            self.merge_to.write( domtree.toxml() )
             self.merge_to.close()
         
         if self.drop_to and len(dropped_nodes) > 0:
@@ -297,5 +198,5 @@
             dropdocroot = dropdoc.documentElement
             for node in dropped_nodes:
                 dropdocroot.appendChild(node)
-            self.drop_to.write( dropdoc.toprettyxml() )
+            self.drop_to.write( dropdoc.toxml() )
             self.drop_to.close()        

Modified: gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py?view=diff&rev=124502&p1=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py&r1=124501&p2=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py&r2=124502
==============================================================================
--- gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py	(original)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py	Fri Jan  7 06:41:28 2005
@@ -40,6 +40,7 @@
     
     node_to_remove_element_from = to_remove.parentNode
     node_to_remove_element_from.removeChild(to_remove)
+    
     if dropped_nodes:
         dropped_nodes.append(to_remove)
 
@@ -87,8 +88,8 @@
     The second argument is merged into the first argument, which is then
     returned.
     """
-    self._import_attributes(target_node, new_node)
-    self._import_children(target_node, new_node)
+    _import_attributes(target_node, new_node)
+    _import_children(target_node, new_node)
 
     
 def _import_attributes(target_node, new_node):
@@ -128,8 +129,8 @@
     def __init__(self, excludedTags):
         self.excludedTags = excludedTags
 
-    def exclude(node):
-        if not child.nodeType == dom.Node.ELEMENT_NODE:
+    def exclude(self, node):
+        if not node.nodeType == dom.Node.ELEMENT_NODE:
             return False
         if node.tagName in self.excludedTags:
             return True
@@ -391,7 +392,7 @@
         self.oldroot = olddoc.documentElement
         self.impl = dom.getDOMImplementation()
         self.newdoc = self.impl.createDocument(None, "workspace", None)
-        self.newroot = newdoc.documentElement
+        self.newroot = self.newdoc.documentElement
         
         self._copy_workspace_root_stuff()
         self._populate_newroot()
@@ -403,6 +404,8 @@
         
         self._normalize_dependencies()
         
+        self._pretty_xml()
+        
         doc = self.newdoc
         # allow GC
         self.repositories = None
@@ -412,7 +415,7 @@
         self.olddoc = None
         self.oldroot = None
         self.newroot = None
-        self.newdoc = none
+        self.newdoc = None
         
         return doc
     
@@ -440,13 +443,13 @@
         
     def _populate_newroot(self):
         """Creates the main containers like <repositories/>."""
-        self.repositories = self.impl.createElement("repositories")
+        self.repositories = self.newdoc.createElement("repositories")
         self.newroot.appendChild(self.repositories)
 
-        self.modules = self.impl.createElement("modules")
+        self.modules = self.newdoc.createElement("modules")
         self.newroot.appendChild(self.modules)
         
-        self.projects = self.impl.createElement("projects")
+        self.projects = self.newdoc.createElement("projects")
         self.newroot.appendChild(self.projects)
     
     def _parse_maven_projects(self):
@@ -500,7 +503,7 @@
         repos = self._get_list_merged_by_name("repository")
         exclude = ["project", "module", "repository"];
         for repo in repos:
-            clone = repo.clone(True)
+            clone = repo.cloneNode(True)
             self._clean_out_by_tag(clone, exclude)
             self.repositories.appendChild(clone)
     
@@ -515,13 +518,13 @@
                 self.log.warn("Dropping module '%s' because no corresponding repository could be found!" % name)
                 continue
             
-            clone = module.clone(True)
+            clone = module.cloneNode(True)
             self._clean_out_by_tag( clone, exclude )
             reporef = self.newdoc.createElement("repository")
             reporef.setAttribute("name", repository.getAttribute("name") )
-            module.appendChild(reporef)
+            clone.insertBefore(reporef, clone.firstChild)
             
-            self.modules.appendChild(module)
+            self.modules.appendChild(clone)
 
     def _find_repository_for_module(self, module):
         repo = None
@@ -538,19 +541,19 @@
         projects = self._get_list_merged_by_name("project")
         exclude = ["project", "module", "repository"];
         for project in projects:
-            module = self._find_module_for_project(module)
+            module = self._find_module_for_project(project)
             if not module:
                 name = project.getAttribute("name")
                 self.log.warn("Dropping project '%s' because no corresponding module could be found!" % name)
                 continue
             
-            clone = project.clone(True)
+            clone = project.cloneNode(True)
             self._clean_out_by_tag( clone, exclude )
             moduleref = self.newdoc.createElement("module")
             moduleref.setAttribute("name", module.getAttribute("name") )
-            project.appendChild(moduleref)
+            clone.insertBefore(moduleref, clone.firstChild)
             
-            self.projects.appendChild(project)
+            self.projects.appendChild(clone)
     
     def _find_module_for_project(self, project):
         repo = None
@@ -565,7 +568,7 @@
 
     def _normalize_dependencies(self):
         """Converts <depend/> and <option/> elements into normalized form."""
-        for project in self.projects:
+        for project in self.projects.getElementsByTagName("project"):
             self._normalize_optional_depend(project)
             dependencies = project.getElementsByTagName("depend")
             if dependencies.length > 0:
@@ -587,7 +590,7 @@
         """Split <depend/> inside <ant/> out into a <depend/> and a <property/>."""
         for dependency in dependencies:
             if dependency.parentNode.tagName in ["ant","maven"]:
-                new_dependency = dependency.clone(True)
+                new_dependency = dependency.cloneNode(True)
                 new_dependency.removeAttribute("property")
                 project.appendChild(new_dependency)
                 
@@ -603,6 +606,7 @@
 
     def _normalize_depend_on_multiple_ids(self, project, dependencies):
         """Split one <depend/> out into multiple, one for each id."""
+        #TODO: reverse that!
         for dependency in dependencies:
             ids = dependency.getAttribute("name")
             if not ids: continue
@@ -611,7 +615,7 @@
             project.removeChild(dependency)
             list = ids.split(",")
             for id in list:
-                new_dependency = dependency.clone(True)
+                new_dependency = dependency.cloneNode(True)
                 new_dependency.setAttribute("ids",id)
                 project.appendChild(new_dependency)
 
@@ -634,11 +638,29 @@
             if newlist.has_key(name):
                 _import_node(newlist[name], elem)
             else:
-                clone = elem.clone(True)
+                clone = elem.cloneNode(True)
                 newlist[name] = clone
         
-        return newlist.values
+        return newlist.values()
+    
+    def _pretty_xml(self):
+        """Adds in some newlines."""
+        
+        self.newroot.insertBefore(self.newdoc.createTextNode("\n"), self.modules)
+        self.newroot.insertBefore(self.newdoc.createTextNode("\n"), self.modules)
+        self.newroot.insertBefore(self.newdoc.createTextNode("\n"), self.modules)
+        self.newroot.insertBefore(self.newdoc.createTextNode("\n"), self.projects)
+        self.newroot.insertBefore(self.newdoc.createTextNode("\n"), self.projects)
+        self.newroot.insertBefore(self.newdoc.createTextNode("\n"), self.projects)
+
+        self.repositories.insertBefore(self.newdoc.createTextNode("\n"), self.repositories.firstChild)
+        self.repositories.appendChild(self.newdoc.createTextNode("\n"))
+
+        self.modules.insertBefore(self.newdoc.createTextNode("\n"), self.modules.firstChild)
+        self.modules.appendChild(self.newdoc.createTextNode("\n"))
 
+        self.projects.insertBefore(self.newdoc.createTextNode("\n"), self.projects.firstChild)
+        self.projects.appendChild(self.newdoc.createTextNode("\n"))
 
 class Objectifier:
     """Turns a *normalized* gump DOM workspace into a pythonified workspace.
@@ -825,6 +847,7 @@
         
     def _create_module(self, module_definition):
         name = module_definition.getAttribute("name")
+        self.log.debug("Converting module definition '%s' into object form." % name)
         repository = self._find_repository_for_module(module_definition)
         
         # parse the attributes and elements common to all modules
@@ -882,6 +905,7 @@
         
     def _create_project(self, project_definition):
         name = project_definition.getAttribute("name")
+        self.log.debug("Converting project definition '%s' into object form." % name)
         module = self._find_module_for_project(project_definition)
         
         project = Project(module, name)
@@ -952,111 +976,6 @@
         project.add_dependency(Dependency(dependency_project,project,optional,runtime,inherit,id))
 
 
-class Visitor:
-    def __init__(self):
-        # we keep a stack of the dependencies of a particular project,
-        # adding an item as we traverse the graph. In the case of a cycle,
-        # we track back through that stack to find it completely
-        self.groups = []
-        
-        # we keep a flat list of all the projects we visit. IF we visit
-        # a project twice, that indicates a cycle, since the topological
-        # sort must have failed
-        self.visited = []
-        
-        # when we find cycles, we store all the projects involved in this
-        # array
-        self.cycles = []
-        
-    def visit(self, project):
-        if project in self.visited:
-            self._find_cycle(project, [project], project)
-        else:
-            self.visited.append(project)
-            self.groups.append(project.dependencies)
-    
-    def done(self, numberOfProjects):
-        # check whether we visited all projects. If not,
-        # the stack in the Verifier was empty before its
-        # time, hence there were projects lying around
-        # with dependencies that weren't satisfied, hence
-        # we must have found a cycle!
-        assert (numberOfProjects > self.visited) == \
-               (len(self.cycles) > 0)
-        
-        if len(self.cycles) > 0:
-            self._handle_cycles()
-    
-    def _find_cycle(self, project, cycle, first):
-        group = self.groups.pop
-        for dependency in group:
-            if dependency.dependee == first:
-                # that completes the cycle
-                self.cycles.append(cycle)
-                break
-            if dependency.dependee == project:
-                # this is the project that references us
-                project_in_cycle = dependency.dependency
-                cycle.append(project_in_cycle)
-                self._handle_cycle(project_in_cycle, cycle, first)
-    
-    def _handle_cycles(self):
-        pass # TODO: remove these projects and their dependendees
-
 class Verifier:
-    """Verifies an objectified gump workspace."""
-
-    def verify(self, workspace):
-        if True: return # TODO
-
-        visitor = Visitor()
-        self.topsortedTravesal(workspace, visitor)
-    
-    def topsortedTravesal(self, workspace, visitor):
-        self._set_indegrees(workspace)
-        # using a stack *should* ensure depth-first
-        stack = self._get_initial_stack(workspace)
-
-        while len(queue) > 0:
-            project = stack.pop
-            visitor.visit(project)
-            
-            for dependency in project.dependencies:
-                dependency.dependency.indegree -= 1
-                if dependency.dependency.indegrees == 0:
-                    stack.append(dependency.dependency)
-        
-        visitor.done(len(workspace.projects))
-        self._clear_indegrees(workspace)
-    
-    def _set_indegrees(projects):
-        """Set the number of in-degrees for each project.
-        
-        The number of in-degrees is a measure of how many
-        dependees a project has. The key bit is that the
-        verifier decreases the number of in-degrees for each
-        project as a dependency is handled.
-        """
-        for project in workspace.projects:
-            project.indegree = 0
-        
-        for dependency in workspace.dependencies:
-            dependency.dependency.indegree += 1
-    
-    def _clear_indegrees(projects):
-        """Removes the in-degrees property from each project."""
-        
-        for project in workspace.proejcts:
-            del project.indegree
-
-    def _get_initial_stack(self, workspace):
-        """Get the projects with an in-degree of 0.
-        
-        In other words, get the projects without dependees.
-        """
-        stack = []
-        for project in workspace.projects:
-            if project.indegree == 0:
-                stack.append(project) 
-        
-        return stack
\ No newline at end of file
+    def verify(self, domtree):
+        pass # TODO!

Added: gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/walker.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/walker.py?view=auto&rev=124502
==============================================================================
--- (empty file)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/walker.py	Fri Jan  7 06:41:28 2005
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module provides facilities for walking a gump model."""
+
+__copyright__ = "Copyright (c) 2005 The Apache Software Foundation"
+__license__   = "http://www.apache.org/licenses/LICENSE-2.0"
+
+class Walker:
+    """Walks a gump tree using topsort.
+
+    Visits projects in dependency order, and just-in-time visits the modules
+    that contain those projects, and just-in-time visits the repositories
+    that contain those projects. Before all that, it visits the workspace
+    itself.
+    
+    Note that the walker will fail to visit all projects if there is a cyclic
+    dependency somewhere.
+    """
+    
+    def walk(self, workspace, visitor):
+        """Walks a gump tree using inverted topsort.
+        
+        Arguments:
+            - workspace -- the workspace to walk
+            - visitor -- the core plugin to send messages to
+
+        Returns a tuple containing the repositories visited, the modules
+        visited, and the projects visited, in the order they were visited.
+        """
+        visitor._visit_workspace(workspace)
+        
+        list = self._topsort_projects(workspace)
+        
+        visited_repositories = []
+        visited_modules = []
+        visited_projects = []
+        
+        for project in list:
+            if not project.module in visited_modules:
+                if not project.module.repository in visited_repositories:
+                    visitor._visit_repository(project.module.repository)
+                    visited_repositories.append(project.module.repository)
+                visitor._visit_module(project.module)
+                visited_modules.append(project.module)
+            
+            visitor._visit_project(project)
+            visited_projects.append(project)
+        
+        return (visited_repositories, visited_modules, visited_projects)
+    
+    def _topsort_projects(self, workspace):
+        """Does a topological sort of the projects in a workspace.
+        
+        The vertices are of course the projects, and the edges are the
+        dependencies between those projects.
+        """
+        self._set_indegrees(workspace)
+        # using a stack *should* ensure depth-first
+        stack = self._get_initial_stack(workspace)
+        
+        list = []
+
+        while len(stack) > 0:
+            project = stack.pop()
+            list.append(project)
+            
+            for dependency in project.dependencies:
+                dependency.dependee.indegree -= 1
+                if dependency.dependee.indegrees == 0:
+                    stack.append(dependency.dependee)
+    
+        self._clear_indegrees(workspace)
+        return list
+
+    def _set_indegrees(self, workspace):
+        """Set the number of in-degrees for each project.
+        
+        The number of in-degrees is a measure of how many
+        dependecies a project has. The key bit is that the
+        walker decreases the number of in-degrees for each
+        project as a dependee is handled.
+        """
+        for project in workspace.projects.values():
+            project.indegree = 0
+        
+        for dependency in workspace.dependencies:
+            dependency.dependee.indegree += 1
+    
+    def _clear_indegrees(self, workspace):
+        """Removes the in-degrees property from each project."""
+        
+        for project in workspace.projects.values():
+            del project.indegree
+
+    def _get_initial_stack(self, workspace):
+        """Get the projects with an in-degree of 0.
+        
+        In other words, get the projects without dependecies.
+        """
+        stack = []
+        for project in workspace.projects.values():
+            if project.indegree == 0:
+                stack.append(project) 
+        
+        return stack    

Modified: gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/__init__.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/__init__.py?view=diff&rev=124502&p1=gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/__init__.py&r1=124501&p2=gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/__init__.py&r2=124502
==============================================================================
--- gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/__init__.py	(original)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/__init__.py	Fri Jan  7 06:41:28 2005
@@ -18,3 +18,115 @@
 
 __copyright__ = "Copyright (c) 2004-2005 The Apache Software Foundation"
 __license__   = "http://www.apache.org/licenses/LICENSE-2.0"
+
+class BaseErrorHandler:
+    """Base error handler for use with the MulticastPlugin.
+    
+    This handler just re-raises a caught error.
+    """
+    def handle(visitor, error, visited_model_object):
+        """Override this method to be able to swallow exceptions."""
+        raise error
+
+class LoggingErrorHandler:
+    """Logging error handler for use with the MulticastPlugin.
+    
+    This handler logs then just re-raises a caught error.
+    """
+    def __init__(self, log):
+        self.log = log
+
+    def handle(visitor, error, visited_model_object):
+        """Override this method to be able to swallow exceptions."""
+        self.log.exception("%s threw an exception while visiting %s!" % (visitor, visited_model_object))
+        raise error
+
+class AbstractPlugin:
+    """Base class for all plugins.
+    
+    To create a concrete plugin, implement one or more of these methods:
+        
+        - visit_workspace(workspace)
+        - visit_repository(repository)
+        - visit_module(module)
+        - visit_project(project)
+    
+    Each of these methods will be called in a "topologically sorted" order.
+    Concretely, this means that:
+        
+        * visit_workspace will be called first;
+        * visit_repository will be called before any contained module or
+          project is visited;
+        * visit_module will be called before any contained project is
+          visited;
+        * visit_project will be called only after all dependencies of that
+          project have already been visited.
+    
+    More concretely, this means plugins usually do not have to worry about
+    the correct ordering of events, since this is usually what you want.
+    """
+    def __init__(self, log):
+        self.log = log
+    
+    def _visit_workspace(self, workspace):
+        if not hasattr(self,'visit_workspace'): return        
+        if not callable(self.visit_workspace): return        
+        self.visit_workspace(workspace)
+    
+    def _visit_repository(self, repository):
+        if not hasattr(self,'visit_repository'): return        
+        if not callable(self.visit_repository): return        
+        self.visit_repository(repository)
+    
+    def _visit_module(self, module):
+        if not hasattr(self,'visit_module'): return        
+        if not callable(self.visit_module): return        
+        self.visit_module(module)
+    
+    def _visit_project(self, project):
+        if not hasattr(self,'visit_project'): return        
+        if not callable(self.visit_project): return        
+        self.visit_project(project)
+
+class MulticastPlugin(AbstractPlugin):
+    """Core plugin that redirects visit_XXX calls to other plugins."""
+    def __init__(self, plugin_list, error_handler=BaseErrorHandler()):
+        self.list = plugin_list
+        self.error_handler = error_handler
+    
+    def visit_workspace(self, workspace):
+        for visitor in self.list:
+            try: visitor._visit_workspace(workspace)
+            except: self.error_handler.handle(visitor, error, workspace)
+
+    def visit_repository(self, repository):
+        for visitor in self.list:
+            try: visitor._visit_repository(repository)
+            except: self.error_handler.handle(visitor, error, repository)
+
+    def visit_module(self, module):
+        for visitor in self.list:
+            try: visitor._visit_module(module)
+            except: self.error_handler.handle(visitor, error, module)
+
+    def visit_project(self, project):
+        for visitor in self.list:
+            try: visitor._visit_project(project)
+            except: self.error_handler.handle(visitor, error, project)
+
+class LoggingPlugin(AbstractPlugin):
+    """Plugin that prints debug messages as it visits model objects."""
+    def __init__(self, log):
+        self.log = log
+    
+    def visit_workspace(self, workspace):
+        self.log.debug("Visiting workspace.")
+    
+    def visit_repository(self, repository):
+        self.log.debug("Visiting repository '%s'." % repository.name)
+    
+    def visit_module(self, module):
+        self.log.debug("Visiting module '%s'." % module.name)
+    
+    def visit_project(self, project):
+        self.log.debug("Visiting project '%s'." % project.name)

Modified: gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/dynagumper.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/dynagumper.py?view=diff&rev=124502&p1=gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/dynagumper.py&r1=124501&p2=gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/dynagumper.py&r2=124502
==============================================================================
--- gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/dynagumper.py	(original)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/plugins/dynagumper.py	Fri Jan  7 06:41:28 2005
@@ -19,29 +19,25 @@
 
 import platform
 
-class Dynagumper:
+from gump.plugins import AbstractPlugin
+
+class Dynagumper(AbstractPlugin):
     """
     Populate the DynaGump run metadata database.
     """
     
-    def __init__(self,run,db, log = None):
-        """
-        Set up the Dynagumper:
-            Dynagumper(run,database)
-        
-        Run is an instance of GumpRun, db is an instance of Database.
-        Optional argument: log, instance of logging.logger or similar.
-        Will use log from gump.logging if not provided.
+    def __init__(self, db, log):
+        """Set up the Dynagumper.
+
+        Arguments:
+            - db is an instance of a gump.util.Database.
+            - log is an instance of logging.Logger.
         """
         self.db = db
-        
-        if not log: from gump import log
         self.log = log
     
     def ensureThisHostIsInDatabase(self):
-        """
-        Adds information about this server to the hosts table.
-        """
+        """Adds information about this server to the hosts table."""
         (system, host, release, version, machine, processor) = platform.uname()
         tablename = "hosts"
         description = "%s (%s,%s,%s,%s,%s)" % (host, system, release, version, machine, processor)
@@ -50,28 +46,15 @@
         
         self.db.execute(cmd)
         
-    def processOtherEvent(self,event):
-        #TODO do the actual work right here...
-        self.log.warning('dynagumper.py processOtherEvent: need to implement event processing')
-                      
-    def processWorkspace(self):
-        """
-        Add information about the workspace to the database.
-        """
+    def visit_workspace(self, workspace):
+        """Add information about the workspace to the database."""
         #TODO do the actual work right here...
-        #self.ensureThisHostIsInDatabase()
-        self.log.warning('dynagumper.py processWorkspace: need to implement workspace event processing')
+        #TODO call ensureThisHostIsInDatabase
     
-    def processModule(self,module):    
-        """
-        Add information about a module to the database.
-        """
+    def visit_module(self, module):    
+        """Add information about a module to the database."""
         #TODO do the actual work
-        self.log.warning('dynagumper.py processModule: need to implement module event processing')
     
-    def processProject(self,project):    
-        """
-        Add information about a project to the database.
-        """
+    def visit_project(self, project):    
+        """Add information about a project to the database."""
         #TODO do the actual work right here...
-        self.log.warning('dynagumper.py processProject: need to implement project event processing')

Mime
View raw message