Mailing-List: contact dev-help@ariatosca.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@ariatosca.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: emblemparade@apache.org
To: dev@ariatosca.incubator.apache.org
Date: Tue, 09 May 2017 15:36:43 -0000
Message-Id: <32b8b00ef3734dd7bee996fd944991a9@git.apache.org>
In-Reply-To: <0adbd02d050a4e23aba234edd3dc44e1@git.apache.org>
References: <0adbd02d050a4e23aba234edd3dc44e1@git.apache.org>
Subject: [03/10] incubator-ariatosca git commit: ARIA-215 Refactor
 plugin-related code into PluginManager
archived-at: Tue, 09 May 2017 15:36:48 -0000

ARIA-215 Refactor plugin-related code into PluginManager

Refactored plugin-related code from ProcessExecutor into PluginManager.
Additionally, renamed plugin_prefix to plugin_dir in PluginManager.


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/3e1ed14c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/3e1ed14c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/3e1ed14c

Branch: refs/heads/ARIA-139-attributes
Commit: 3e1ed14c00ea2c83fafdca8ec0e37817bea1d5e8
Parents: 45c158e
Author: Ran Ziv <ran@gigaspaces.com>
Authored: Sun May 7 16:36:39 2017 +0300
Committer: Ran Ziv <ran@gigaspaces.com>
Committed: Sun May 7 17:27:36 2017 +0300

----------------------------------------------------------------------
 aria/orchestrator/plugin.py                     | 36 +++++++++++-
 aria/orchestrator/workflows/executor/process.py | 60 ++++++--------------
 aria/utils/process.py                           | 47 +++++++++++++++
 tests/utils/test_plugin.py                      |  6 +-
 4 files changed, 100 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/orchestrator/plugin.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/plugin.py b/aria/orchestrator/plugin.py
index f99666c..8fbcf5a 100644
--- a/aria/orchestrator/plugin.py
+++ b/aria/orchestrator/plugin.py
@@ -23,6 +23,9 @@ from datetime import datetime
 import wagon
 
 from . import exceptions
+from ..utils import process as process_utils
+
+_IS_WIN = os.name == 'nt'
 
 
 class PluginManager(object):
@@ -62,11 +65,40 @@ class PluginManager(object):
             raise exceptions.PluginAlreadyExistsError(
                 'Plugin {0}, version {1} already exists'.format(plugin.package_name,
                                                                 plugin.package_version))
-        self._install_wagon(source=source, prefix=self.get_plugin_prefix(plugin))
+        self._install_wagon(source=source, prefix=self.get_plugin_dir(plugin))
         self._model.plugin.put(plugin)
         return plugin
 
-    def get_plugin_prefix(self, plugin):
+    def load_plugin(self, plugin, env=None):
+        """
+        Load the plugin into an environment.
+        Loading the plugin means the plugin's code and binaries paths will be appended to the
+        environment's PATH and PYTHONPATH, thereby allowing usage of the plugin.
+        :param plugin: The plugin to load
+        :param env: The environment to load the plugin into; If `None`, os.environ will be used.
+        """
+        env = env or os.environ
+        plugin_dir = self.get_plugin_dir(plugin)
+
+        # Update PATH environment variable to include plugin's bin dir
+        bin_dir = 'Scripts' if _IS_WIN else 'bin'
+        process_utils.append_to_path(os.path.join(plugin_dir, bin_dir), env=env)
+
+        # Update PYTHONPATH environment variable to include plugin's site-packages
+        # directories
+        if _IS_WIN:
+            pythonpath_dirs = [os.path.join(plugin_dir, 'Lib', 'site-packages')]
+        else:
+            # In some linux environments, there will be both a lib and a lib64 directory
+            # with the latter, containing compiled packages.
+            pythonpath_dirs = [os.path.join(
+                plugin_dir, 'lib{0}'.format(b),
+                'python{0}.{1}'.format(sys.version_info[0], sys.version_info[1]),
+                'site-packages') for b in ('', '64')]
+
+        process_utils.append_to_pythonpath(*pythonpath_dirs, env=env)
+
+    def get_plugin_dir(self, plugin):
         return os.path.join(
             self._plugins_dir,
             '{0}-{1}'.format(plugin.package_name, plugin.package_version))

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index 8481406..f3daf04 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -47,13 +47,12 @@ from aria.storage import instrumentation
 from aria.extension import process_executor
 from aria.utils import (
     imports,
-    exceptions
+    exceptions,
+    process as process_utils
 )
 from aria.modeling import types as modeling_types
 
 
-_IS_WIN = os.name == 'nt'
-
 _INT_FMT = 'I'
 _INT_SIZE = struct.calcsize(_INT_FMT)
 UPDATE_TRACKED_CHANGES_FAILED_STR = \
@@ -127,13 +126,7 @@ class ProcessExecutor(base.BaseExecutor):
         with open(arguments_json_path, 'wb') as f:
             f.write(pickle.dumps(self._create_arguments_dict(task)))
 
-        env = os.environ.copy()
-        # See _update_env for plugin_prefix usage
-        if task.plugin_fk and self._plugin_manager:
-            plugin_prefix = self._plugin_manager.get_plugin_prefix(task.plugin)
-        else:
-            plugin_prefix = None
-        self._update_env(env=env, plugin_prefix=plugin_prefix)
+        env = self._construct_subprocess_env(task=task)
         # Asynchronously start the operation in a subprocess
         subprocess.Popen(
             '{0} {1} {2}'.format(sys.executable, __file__, arguments_json_path),
@@ -156,40 +149,19 @@ class ProcessExecutor(base.BaseExecutor):
             'context': task.context.serialization_dict,
         }
 
-    def _update_env(self, env, plugin_prefix):
-        pythonpath_dirs = []
-        # If this is a plugin operation, plugin prefix will point to where
-        # This plugin is installed.
-        # We update the environment variables that the subprocess will be started with based on it
-        if plugin_prefix:
-
-            # Update PATH environment variable to include plugin's bin dir
-            bin_dir = 'Scripts' if _IS_WIN else 'bin'
-            env['PATH'] = '{0}{1}{2}'.format(
-                os.path.join(plugin_prefix, bin_dir),
-                os.pathsep,
-                env.get('PATH', ''))
-
-            # Update PYTHONPATH environment variable to include plugin's site-packages
-            # directories
-            if _IS_WIN:
-                pythonpath_dirs = [os.path.join(plugin_prefix, 'Lib', 'site-packages')]
-            else:
-                # In some linux environments, there will be both a lib and a lib64 directory
-                # with the latter, containing compiled packages.
-                pythonpath_dirs = [os.path.join(
-                    plugin_prefix, 'lib{0}'.format(b),
-                    'python{0}.{1}'.format(sys.version_info[0], sys.version_info[1]),
-                    'site-packages') for b in ('', '64')]
-
-        # Add used supplied directories to injected PYTHONPATH
-        pythonpath_dirs.extend(self._python_path)
-
-        if pythonpath_dirs:
-            env['PYTHONPATH'] = '{0}{1}{2}'.format(
-                os.pathsep.join(pythonpath_dirs),
-                os.pathsep,
-                env.get('PYTHONPATH', ''))
+    def _construct_subprocess_env(self, task):
+        env = os.environ.copy()
+
+        if task.plugin_fk and self._plugin_manager:
+            # If this is a plugin operation,
+            # load the plugin on the subprocess env we're constructing
+            self._plugin_manager.load_plugin(task.plugin, env=env)
+
+        # Add user supplied directories to injected PYTHONPATH
+        if self._python_path:
+            process_utils.append_to_pythonpath(*self._python_path, env=env)
+
+        return env
 
     def _listener(self):
         # Notify __init__ method this thread has actually started

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/utils/process.py
----------------------------------------------------------------------
diff --git a/aria/utils/process.py b/aria/utils/process.py
new file mode 100644
index 0000000..9aeae67
--- /dev/null
+++ b/aria/utils/process.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+
+def append_to_path(*args, **kwargs):
+    """
+    Appends one or more paths to the system path of an environment.
+    The environment will be that of the current process unless another is passed using the
+    'env' keyword argument.
+    :param args: paths to append
+    :param kwargs: 'env' may be used to pass a custom environment to use
+    """
+    _append_to_path('PATH', *args, **kwargs)
+
+
+def append_to_pythonpath(*args, **kwargs):
+    """
+    Appends one or more paths to the python path of an environment.
+    The environment will be that of the current process unless another is passed using the
+    'env' keyword argument.
+    :param args: paths to append
+    :param kwargs: 'env' may be used to pass a custom environment to use
+    """
+    _append_to_path('PYTHONPATH', *args, **kwargs)
+
+
+def _append_to_path(path, *args, **kwargs):
+    env = kwargs.get('env') or os.environ
+    env[path] = '{0}{1}{2}'.format(
+        os.pathsep.join(args),
+        os.pathsep,
+        env.get(path, '')
+    )

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/tests/utils/test_plugin.py
----------------------------------------------------------------------
diff --git a/tests/utils/test_plugin.py b/tests/utils/test_plugin.py
index 3350247..c91d0c9 100644
--- a/tests/utils/test_plugin.py
+++ b/tests/utils/test_plugin.py
@@ -38,9 +38,9 @@ class TestPluginManager(object):
         assert plugin.package_name == PACKAGE_NAME
         assert plugin.package_version == PACKAGE_VERSION
         assert plugin == model.plugin.get(plugin.id)
-        plugin_prefix = os.path.join(plugins_dir, '{0}-{1}'.format(PACKAGE_NAME, PACKAGE_VERSION))
-        assert os.path.isdir(plugin_prefix)
-        assert plugin_prefix == plugin_manager.get_plugin_prefix(plugin)
+        plugin_dir = os.path.join(plugins_dir, '{0}-{1}'.format(PACKAGE_NAME, PACKAGE_VERSION))
+        assert os.path.isdir(plugin_dir)
+        assert plugin_dir == plugin_manager.get_plugin_dir(plugin)
 
     def test_install_already_exits(self, plugin_manager, mock_plugin):
         plugin_manager.install(mock_plugin)