Return-Path: X-Original-To: apmail-aurora-commits-archive@minotaur.apache.org Delivered-To: apmail-aurora-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9E27D1813D for ; Mon, 13 Jul 2015 23:12:03 +0000 (UTC) Received: (qmail 11124 invoked by uid 500); 13 Jul 2015 23:12:03 -0000 Delivered-To: apmail-aurora-commits-archive@aurora.apache.org Received: (qmail 11088 invoked by uid 500); 13 Jul 2015 23:12:03 -0000 Mailing-List: contact commits-help@aurora.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@aurora.apache.org Delivered-To: mailing list commits@aurora.apache.org Received: (qmail 11079 invoked by uid 99); 13 Jul 2015 23:12:03 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Jul 2015 23:12:03 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 37279E0AD6; Mon, 13 Jul 2015 23:12:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wickman@apache.org To: commits@aurora.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: aurora git commit: Fix AuroraConfigLoader schema filtering when loading from string. Date: Mon, 13 Jul 2015 23:12:03 +0000 (UTC) Repository: aurora Updated Branches: refs/heads/master b7a02a5a6 -> 703c9fb78 Fix AuroraConfigLoader schema filtering when loading from string. We have an internal tool that needs to serialize configs into a string and retrieve them from stable storage. We had deployed a client containing shutdown_endpoint then reverted it in https://reviews.apache.org/r/35847/ but unfortunately deserialization broke the internal tool since it was not filtering out the unknown attributes. Testing Done: Added test, it broke. Fixed code, tests passed. Also updated tests to not create files when not necessary. Reviewed at https://reviews.apache.org/r/36459/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/703c9fb7 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/703c9fb7 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/703c9fb7 Branch: refs/heads/master Commit: 703c9fb78d452be861fae90acddbcab0c6c0b723 Parents: b7a02a5 Author: Brian Wickman Authored: Mon Jul 13 16:11:57 2015 -0700 Committer: Brian Wickman Committed: Mon Jul 13 16:11:57 2015 -0700 ---------------------------------------------------------------------- src/main/python/apache/aurora/config/loader.py | 3 +- .../python/apache/aurora/config/test_loader.py | 40 +++++++++----------- 2 files changed, 19 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/703c9fb7/src/main/python/apache/aurora/config/loader.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/config/loader.py b/src/main/python/apache/aurora/config/loader.py index c8b045e..a967b9d 100644 --- a/src/main/python/apache/aurora/config/loader.py +++ b/src/main/python/apache/aurora/config/loader.py @@ -12,7 +12,6 @@ # limitations under the License. # -import json import pkgutil from pystachio.config import Config as PystachioConfig @@ -70,7 +69,7 @@ class AuroraConfigLoader(PystachioConfig): @classmethod def loads_json(cls, string): - return base_schema.Job(json.loads(string)) + return base_schema.Job.json_loads(string) AuroraConfigLoader.flush_schemas() http://git-wip-us.apache.org/repos/asf/aurora/blob/703c9fb7/src/test/python/apache/aurora/config/test_loader.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/config/test_loader.py b/src/test/python/apache/aurora/config/test_loader.py index 00b6eab..9789255 100644 --- a/src/test/python/apache/aurora/config/test_loader.py +++ b/src/test/python/apache/aurora/config/test_loader.py @@ -14,6 +14,7 @@ import json import tempfile +from io import BytesIO import pytest from twitter.common.contextutil import temporary_file @@ -49,30 +50,28 @@ def test_enoent(): def test_bad_config(): - with temporary_file() as fp: - fp.write(BAD_MESOS_CONFIG) - fp.flush() - with pytest.raises(AuroraConfigLoader.InvalidConfigError): - AuroraConfigLoader.load(fp.name) + with pytest.raises(AuroraConfigLoader.InvalidConfigError): + AuroraConfigLoader.load(BytesIO(BAD_MESOS_CONFIG)) + + +def test_filter_schema(): + env = AuroraConfigLoader.load(BytesIO(MESOS_CONFIG)) + job_dict = env['jobs'][0].get() + job_dict['unknown_attribute'] = 'foo bar baz' + job_json_string = json.dumps(job_dict) + # If this fails, will raise an InvalidConfigError or other exception and fail the test. + AuroraConfigLoader.loads_json(job_json_string) def test_empty_config(): - with temporary_file() as fp: - fp.flush() - AuroraConfigLoader.load(fp.name) + AuroraConfigLoader.load(BytesIO()) def test_load_json(): - with temporary_file() as fp: - fp.write(MESOS_CONFIG) - fp.flush() - env = AuroraConfigLoader.load(fp.name) - job = env['jobs'][0] - with temporary_file() as fp: - fp.write(json.dumps(job.get())) - fp.flush() - new_job = AuroraConfigLoader.load_json(fp.name) - assert new_job == job + env = AuroraConfigLoader.load(BytesIO(MESOS_CONFIG)) + job = env['jobs'][0] + new_job = AuroraConfigLoader.loads_json(json.dumps(job.get())) + assert new_job == job def test_load(): @@ -89,10 +88,7 @@ def test_load(): def test_pick(): - with temporary_file() as fp: - fp.write(MESOS_CONFIG) - fp.flush() - env = AuroraConfigLoader.load(fp.name) + env = AuroraConfigLoader.load(BytesIO(MESOS_CONFIG)) hello_world = env['jobs'][0] assert AuroraConfig.pick(env, 'hello_world', None) == hello_world