From commits-return-13233-archive-asf-public=cust-asf.ponee.io@airflow.incubator.apache.org Wed Mar 14 09:15:46 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id C5966180654 for ; Wed, 14 Mar 2018 09:15:45 +0100 (CET) Received: (qmail 23458 invoked by uid 500); 14 Mar 2018 08:15:44 -0000 Mailing-List: contact commits-help@airflow.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airflow.incubator.apache.org Delivered-To: mailing list commits@airflow.incubator.apache.org Received: (qmail 23449 invoked by uid 99); 14 Mar 2018 08:15:44 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Mar 2018 08:15:44 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 56A3C180160 for ; Wed, 14 Mar 2018 08:15:44 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -11.731 X-Spam-Level: X-Spam-Status: No, score=-11.731 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, SPF_PASS=-0.001, T_RP_MATCHES_RCVD=-0.01, USER_IN_DEF_SPF_WL=-7.5] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id HwVqOID8EhOX for ; Wed, 14 Mar 2018 08:15:42 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 8B5D25F503 for ; Wed, 14 Mar 2018 08:15:41 +0000 (UTC) Received: (qmail 22740 invoked by uid 99); 14 Mar 2018 08:15:40 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Mar 2018 08:15:40 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 69202E96E4; Wed, 14 Mar 2018 08:15:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: fokko@apache.org To: commits@airflow.incubator.apache.org Date: Wed, 14 Mar 2018 08:15:43 -0000 Message-Id: <75fa2ab54f3c4a3fb53a87b45b43e672@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [4/6] incubator-airflow git commit: [AIRFLOW-2203] Cache signature in apply_defaults [AIRFLOW-2203] Cache signature in apply_defaults Cache inspect.signature for the wrapper closure to avoid calling it at every decorated invocation. This is separate sig_cache created per decoration, i.e. each function decorated using apply_defaults will have a different sig_cache. Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/81ec595b Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/81ec595b Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/81ec595b Branch: refs/heads/master Commit: 81ec595b6c1ac05bc7f42e2c92c0dd79409953a4 Parents: 92357d5 Author: wongwill86 Authored: Mon Mar 12 17:08:44 2018 -0400 Committer: Fokko Driesprong Committed: Wed Mar 14 09:11:50 2018 +0100 ---------------------------------------------------------------------- airflow/utils/decorators.py | 31 ++++++++++------- tests/utils/test_decorators.py | 69 +++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/81ec595b/airflow/utils/decorators.py ---------------------------------------------------------------------- diff --git a/airflow/utils/decorators.py b/airflow/utils/decorators.py index 995e60f..4e9cb05 100644 --- a/airflow/utils/decorators.py +++ b/airflow/utils/decorators.py @@ -39,6 +39,19 @@ def apply_defaults(func): inheritance and argument defaults, this decorator also alerts with specific information about the missing arguments. """ + + import airflow.models + # Cache inspect.signature for the wrapper closure to avoid calling it + # at every decorated invocation. This is separate sig_cache created + # per decoration, i.e. each function decorated using apply_defaults will + # have a different sig_cache. + sig_cache = signature(func) + non_optional_args = { + name for (name, param) in sig_cache.parameters.items() + if param.default == param.empty and + param.name != 'self' and + param.kind not in (param.VAR_POSITIONAL, param.VAR_KEYWORD)} + @wraps(func) def wrapper(*args, **kwargs): if len(args) > 1: @@ -46,9 +59,9 @@ def apply_defaults(func): "Use keyword arguments when initializing operators") dag_args = {} dag_params = {} - import airflow.models - if kwargs.get('dag', None) or airflow.models._CONTEXT_MANAGER_DAG: - dag = kwargs.get('dag', None) or airflow.models._CONTEXT_MANAGER_DAG + + dag = kwargs.get('dag', None) or airflow.models._CONTEXT_MANAGER_DAG + if dag: dag_args = copy(dag.default_args) or {} dag_params = copy(dag.params) or {} @@ -67,16 +80,10 @@ def apply_defaults(func): dag_args.update(default_args) default_args = dag_args - sig = signature(func) - non_optional_args = [ - name for (name, param) in sig.parameters.items() - if param.default == param.empty and - param.name != 'self' and - param.kind not in (param.VAR_POSITIONAL, param.VAR_KEYWORD)] - for arg in sig.parameters: - if arg in default_args and arg not in kwargs: + for arg in sig_cache.parameters: + if arg not in kwargs and arg in default_args: kwargs[arg] = default_args[arg] - missing_args = list(set(non_optional_args) - set(kwargs)) + missing_args = list(non_optional_args - set(kwargs)) if missing_args: msg = "Argument {0} is required".format(missing_args) raise AirflowException(msg) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/81ec595b/tests/utils/test_decorators.py ---------------------------------------------------------------------- diff --git a/tests/utils/test_decorators.py b/tests/utils/test_decorators.py new file mode 100644 index 0000000..29dada7 --- /dev/null +++ b/tests/utils/test_decorators.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from airflow.utils.decorators import apply_defaults +from airflow.exceptions import AirflowException + + +# Essentially similar to airflow.models.BaseOperator +class DummyClass(object): + @apply_defaults + def __init__(self, test_param, params=None, default_args=None): + self.test_param = test_param + + +class DummySubClass(DummyClass): + @apply_defaults + def __init__(self, test_sub_param, *args, **kwargs): + super(DummySubClass, self).__init__(*args, **kwargs) + self.test_sub_param = test_sub_param + + +class ApplyDefaultTest(unittest.TestCase): + + def test_apply(self): + dc = DummyClass(test_param=True) + self.assertTrue(dc.test_param) + + with self.assertRaisesRegexp(AirflowException, 'Argument.*test_param.*required'): + DummySubClass(test_sub_param=True) + + def test_default_args(self): + default_args = {'test_param': True} + dc = DummyClass(default_args=default_args) + self.assertTrue(dc.test_param) + + default_args = {'test_param': True, 'test_sub_param': True} + dsc = DummySubClass(default_args=default_args) + self.assertTrue(dc.test_param) + self.assertTrue(dsc.test_sub_param) + + default_args = {'test_param': True} + dsc = DummySubClass(default_args=default_args, test_sub_param=True) + self.assertTrue(dc.test_param) + self.assertTrue(dsc.test_sub_param) + + with self.assertRaisesRegexp(AirflowException, + 'Argument.*test_sub_param.*required'): + DummySubClass(default_args=default_args) + + def test_incorrect_default_args(self): + default_args = {'test_param': True, 'extra_param': True} + dc = DummyClass(default_args=default_args) + self.assertTrue(dc.test_param) + + default_args = {'random_params': True} + with self.assertRaisesRegexp(AirflowException, 'Argument.*test_param.*required'): + DummyClass(default_args=default_args)