beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rober...@apache.org
Subject [2/4] incubator-beam git commit: Make save_main_session optional
Date Sat, 23 Jul 2016 23:43:06 GMT
Make save_main_session optional


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2e36602c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2e36602c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2e36602c

Branch: refs/heads/python-sdk
Commit: 2e36602c0a2944129698866cc31abc3bcac6169f
Parents: af6d380
Author: Silviu Calinoiu <silviuc@google.com>
Authored: Sat Jul 23 09:31:30 2016 -0700
Committer: Silviu Calinoiu <silviuc@google.com>
Committed: Sat Jul 23 09:31:30 2016 -0700

----------------------------------------------------------------------
 .../python/apache_beam/utils/dependency_test.py | 33 ++++++++++++--------
 sdks/python/apache_beam/utils/options.py        |  2 +-
 2 files changed, 21 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2e36602c/sdks/python/apache_beam/utils/dependency_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py
index b7a9296..2f9a57b 100644
--- a/sdks/python/apache_beam/utils/dependency_test.py
+++ b/sdks/python/apache_beam/utils/dependency_test.py
@@ -81,10 +81,12 @@ class SetupTest(unittest.TestCase):
         [],
         dependency.stage_job_resources(options))
 
-  def test_default_resources(self):
+  def test_with_main_session(self):
     staging_dir = tempfile.mkdtemp()
     options = PipelineOptions()
+
     options.view_as(GoogleCloudOptions).staging_location = staging_dir
+    options.view_as(SetupOptions).save_main_session = True
     self.update_options(options)
 
     self.assertEqual(
@@ -94,6 +96,16 @@ class SetupTest(unittest.TestCase):
         os.path.isfile(
             os.path.join(staging_dir, names.PICKLED_MAIN_SESSION_FILE)))
 
+  def test_default_resources(self):
+    staging_dir = tempfile.mkdtemp()
+    options = PipelineOptions()
+    options.view_as(GoogleCloudOptions).staging_location = staging_dir
+    self.update_options(options)
+
+    self.assertEqual(
+        [],
+        dependency.stage_job_resources(options))
+
   def test_with_requirements_file(self):
     staging_dir = tempfile.mkdtemp()
     source_dir = tempfile.mkdtemp()
@@ -106,7 +118,7 @@ class SetupTest(unittest.TestCase):
     self.create_temp_file(
         os.path.join(source_dir, dependency.REQUIREMENTS_FILE), 'nothing')
     self.assertEqual(
-        sorted([dependency.REQUIREMENTS_FILE, names.PICKLED_MAIN_SESSION_FILE,
+        sorted([dependency.REQUIREMENTS_FILE,
                 'abc.txt', 'def.txt']),
         sorted(dependency.stage_job_resources(
             options,
@@ -145,7 +157,7 @@ class SetupTest(unittest.TestCase):
     self.create_temp_file(
         os.path.join(source_dir, dependency.REQUIREMENTS_FILE), 'nothing')
     self.assertEqual(
-        sorted([dependency.REQUIREMENTS_FILE, names.PICKLED_MAIN_SESSION_FILE,
+        sorted([dependency.REQUIREMENTS_FILE,
                 'abc.txt', 'def.txt']),
         sorted(dependency.stage_job_resources(
             options,
@@ -169,8 +181,7 @@ class SetupTest(unittest.TestCase):
         source_dir, 'setup.py')
 
     self.assertEqual(
-        [dependency.WORKFLOW_TARBALL_FILE,
-         names.PICKLED_MAIN_SESSION_FILE],
+        [dependency.WORKFLOW_TARBALL_FILE],
         dependency.stage_job_resources(
             options,
             # We replace the build setup command because a realistic one would
@@ -265,8 +276,7 @@ class SetupTest(unittest.TestCase):
     options.view_as(SetupOptions).sdk_location = 'default'
 
     self.assertEqual(
-        [names.PICKLED_MAIN_SESSION_FILE,
-         names.DATAFLOW_SDK_TARBALL_FILE],
+        [names.DATAFLOW_SDK_TARBALL_FILE],
         dependency.stage_job_resources(
             options,
             file_copy=dependency._dependency_file_copy))
@@ -286,8 +296,7 @@ class SetupTest(unittest.TestCase):
     options.view_as(SetupOptions).sdk_location = sdk_location
 
     self.assertEqual(
-        [names.PICKLED_MAIN_SESSION_FILE,
-         names.DATAFLOW_SDK_TARBALL_FILE],
+        [names.DATAFLOW_SDK_TARBALL_FILE],
         dependency.stage_job_resources(options))
     tarball_path = os.path.join(
         staging_dir, names.DATAFLOW_SDK_TARBALL_FILE)
@@ -321,8 +330,7 @@ class SetupTest(unittest.TestCase):
     options.view_as(SetupOptions).sdk_location = sdk_location
 
     self.assertEqual(
-        [names.PICKLED_MAIN_SESSION_FILE,
-         names.DATAFLOW_SDK_TARBALL_FILE],
+        [names.DATAFLOW_SDK_TARBALL_FILE],
         dependency.stage_job_resources(options))
 
   def test_with_extra_packages(self):
@@ -363,8 +371,7 @@ class SetupTest(unittest.TestCase):
 
     self.assertEqual(
         ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'gcs.tar.gz',
-         dependency.EXTRA_PACKAGES_FILE,
-         names.PICKLED_MAIN_SESSION_FILE],
+         dependency.EXTRA_PACKAGES_FILE],
         dependency.stage_job_resources(options))
     with open(os.path.join(staging_dir, dependency.EXTRA_PACKAGES_FILE)) as f:
       self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'xyz2.tar\n',

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2e36602c/sdks/python/apache_beam/utils/options.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/utils/options.py b/sdks/python/apache_beam/utils/options.py
index 335beea..933aa12 100644
--- a/sdks/python/apache_beam/utils/options.py
+++ b/sdks/python/apache_beam/utils/options.py
@@ -412,7 +412,7 @@ class SetupOptions(PipelineOptions):
          'install the resulting package before running any custom code.'))
     parser.add_argument(
         '--save_main_session',
-        default=True,
+        default=False,
         action='store_true',
         help=
         ('Save the main session state so that pickled functions and classes '


Mime
View raw message