beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rober...@apache.org
Subject [1/2] incubator-beam git commit: Allow .whl files to be staged with --extra_package
Date Mon, 26 Sep 2016 22:05:39 GMT
Repository: incubator-beam
Updated Branches:
  refs/heads/python-sdk 8a333a661 -> 19e3eff91


Allow .whl files to be staged with --extra_package


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/71c474a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/71c474a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/71c474a5

Branch: refs/heads/python-sdk
Commit: 71c474a5d7227be9391388fda41875305b952f93
Parents: 8a333a6
Author: Charles Chen <ccy@google.com>
Authored: Thu Sep 22 20:19:05 2016 -0700
Committer: Robert Bradshaw <robertwb@google.com>
Committed: Mon Sep 26 15:05:00 2016 -0700

----------------------------------------------------------------------
 sdks/python/apache_beam/utils/dependency.py      | 14 +++++++++++---
 sdks/python/apache_beam/utils/dependency_test.py | 12 +++++++-----
 sdks/python/apache_beam/utils/options.py         | 16 ++++++++--------
 3 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/dependency.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py
index 314bd1b..ab311a0 100644
--- a/sdks/python/apache_beam/utils/dependency.py
+++ b/sdks/python/apache_beam/utils/dependency.py
@@ -141,10 +141,18 @@ def _stage_extra_packages(extra_packages, staging_location, temp_dir,
   local_packages = []
   for package in extra_packages:
     if not (os.path.basename(package).endswith('.tar') or
-            os.path.basename(package).endswith('.tar.gz')):
+            os.path.basename(package).endswith('.tar.gz') or
+            os.path.basename(package).endswith('.whl')):
       raise RuntimeError(
-          'The --extra_packages option expects a full path ending with '
-          '\'.tar\' or \'.tar.gz\' instead of %s' % package)
+          'The --extra_package option expects a full path ending with '
+          '".tar" or ".tar.gz" instead of %s' % package)
+    if os.path.basename(package).endswith('.whl'):
+      logging.warning(
+          'The .whl package "%s" is provided in --extra_package. '
+          'This functionality is not officially supported. Since wheel '
+          'packages are binary distributions, this package must be '
+          'binary-compatible with the worker environment (e.g. Python 2.7 '
+          'running on an x64 Linux host).')
 
     if not os.path.isfile(package):
       if package.startswith('gs://'):

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/dependency_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py
index ca31806..3549a07 100644
--- a/sdks/python/apache_beam/utils/dependency_test.py
+++ b/sdks/python/apache_beam/utils/dependency_test.py
@@ -342,6 +342,8 @@ class SetupTest(unittest.TestCase):
     self.create_temp_file(
         os.path.join(source_dir, 'xyz2.tar'), 'nothing')
     self.create_temp_file(
+        os.path.join(source_dir, 'whl.whl'), 'nothing')
+    self.create_temp_file(
         os.path.join(source_dir, dependency.EXTRA_PACKAGES_FILE), 'nothing')
 
     options = PipelineOptions()
@@ -351,6 +353,7 @@ class SetupTest(unittest.TestCase):
         os.path.join(source_dir, 'abc.tar.gz'),
         os.path.join(source_dir, 'xyz.tar.gz'),
         os.path.join(source_dir, 'xyz2.tar'),
+        os.path.join(source_dir, 'whl.whl'),
         'gs://my-gcs-bucket/gcs.tar.gz']
 
     gcs_copied_files = []
@@ -369,12 +372,12 @@ class SetupTest(unittest.TestCase):
     dependency._dependency_file_copy = file_copy
 
     self.assertEqual(
-        ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'gcs.tar.gz',
+        ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'whl.whl', 'gcs.tar.gz',
          dependency.EXTRA_PACKAGES_FILE],
         dependency.stage_job_resources(options))
     with open(os.path.join(staging_dir, dependency.EXTRA_PACKAGES_FILE)) as f:
       self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'xyz2.tar\n',
-                        'gcs.tar.gz\n'], f.readlines())
+                        'whl.whl\n', 'gcs.tar.gz\n'], f.readlines())
     self.assertEqual(['gs://my-gcs-bucket/gcs.tar.gz'], gcs_copied_files)
 
   def test_with_extra_packages_missing_files(self):
@@ -406,9 +409,8 @@ class SetupTest(unittest.TestCase):
       dependency.stage_job_resources(options)
     self.assertEqual(
         cm.exception.message,
-        'The --extra_packages option expects a full path ending with '
-        '\'.tar\' or \'.tar.gz\' instead of %s' % os.path.join(source_dir,
-                                                               'abc.tgz'))
+        'The --extra_package option expects a full path ending with ".tar" or '
+        '".tar.gz" instead of %s' % os.path.join(source_dir, 'abc.tgz'))
 
 
 if __name__ == '__main__':

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/options.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/options.py b/sdks/python/apache_beam/utils/options.py
index 700c080..a964036 100644
--- a/sdks/python/apache_beam/utils/options.py
+++ b/sdks/python/apache_beam/utils/options.py
@@ -442,14 +442,14 @@ class SetupOptions(PipelineOptions):
         action='append',
         default=None,
         help=
-        ('Local path to a Python package file. The file is expected to be a '
-         'compressed tarball with the suffix \'.tar.gz\' which can be '
-         'installed using the easy_install command of the standard setuptools '
-         'package. Multiple --extra_package options can be specified if more '
-         'than one package is needed. During job submission the files will be '
-         'staged in the staging area (--staging_location option) and the '
-         'workers will install them in same order they were specified on the '
-         'command line.'))
+        ('Local path to a Python package file. The file is expected to be (1) '
+         'a package tarball (".tar") or (2) a compressed package tarball '
+         '(".tar.gz") which can be installed using the "pip install" command '
+         'of the standard pip package. Multiple --extra_package options can '
+         'be specified if more than one package is needed. During job '
+         'submission, the files will be staged in the staging area '
+         '(--staging_location option) and the workers will install them in '
+         'same order they were specified on the command line.'))
 
 # TODO(silviuc): Add --files_to_stage option.
 # This could potentially replace the --requirements_file and --setup_file.


Mime
View raw message