Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4B4D22004F3 for ; Tue, 15 Aug 2017 19:22:17 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 49F4A167098; Tue, 15 Aug 2017 17:22:17 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6A434167095 for ; Tue, 15 Aug 2017 19:22:16 +0200 (CEST) Received: (qmail 39366 invoked by uid 500); 15 Aug 2017 17:22:15 -0000 Mailing-List: contact commits-help@airflow.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airflow.incubator.apache.org Delivered-To: mailing list commits@airflow.incubator.apache.org Received: (qmail 39357 invoked by uid 99); 15 Aug 2017 17:22:15 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd2-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Aug 2017 17:22:15 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd2-us-west.apache.org (ASF Mail Server at spamd2-us-west.apache.org) with ESMTP id E17B11A065E for ; Tue, 15 Aug 2017 17:22:14 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd2-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.222 X-Spam-Level: X-Spam-Status: No, score=-4.222 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001, SPF_PASS=-0.001] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd2-us-west.apache.org [10.40.0.9]) (amavisd-new, port 10024) with ESMTP id 5tZKHgTpQqw0 for ; Tue, 15 Aug 2017 17:22:13 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id D03385F3CF for ; Tue, 15 Aug 2017 17:22:12 +0000 (UTC) Received: (qmail 39345 invoked by uid 99); 15 Aug 2017 17:22:12 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Aug 2017 17:22:12 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 47B91E0612; Tue, 15 Aug 2017 17:22:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: criccomini@apache.org To: commits@airflow.incubator.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: incubator-airflow git commit: [AIRFLOW-1504] Log dataproc cluster name Date: Tue, 15 Aug 2017 17:22:12 +0000 (UTC) archived-at: Tue, 15 Aug 2017 17:22:17 -0000 Repository: incubator-airflow Updated Branches: refs/heads/master 42cad6069 -> 1cd6c4b0e [AIRFLOW-1504] Log dataproc cluster name Closes #2517 from TrevorEdwards/dataproc_log_clustername Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/1cd6c4b0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/1cd6c4b0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/1cd6c4b0 Branch: refs/heads/master Commit: 1cd6c4b0e8d73426be49106eaab75ace1bf4a4bf Parents: 42cad60 Author: Trevor Edwards Authored: Tue Aug 15 10:22:03 2017 -0700 Committer: Chris Riccomini Committed: Tue Aug 15 10:22:03 2017 -0700 ---------------------------------------------------------------------- airflow/contrib/operators/dataproc_operator.py | 2 + .../contrib/operators/test_dataproc_operator.py | 109 +++++++++++++++++++ 2 files changed, 111 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1cd6c4b0/airflow/contrib/operators/dataproc_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py index 4b6b61c..aa9b335 100644 --- a/airflow/contrib/operators/dataproc_operator.py +++ b/airflow/contrib/operators/dataproc_operator.py @@ -250,6 +250,7 @@ class DataprocClusterCreateOperator(BaseOperator): return cluster_data def execute(self, context): + logging.info('Creating cluster: {}'.format(self.cluster_name)) hook = DataProcHook( gcp_conn_id=self.google_cloud_conn_id, delegate_to=self.delegate_to @@ -341,6 +342,7 @@ class DataprocClusterDeleteOperator(BaseOperator): time.sleep(15) def execute(self, context): + logging.info('Deleting cluster: {}'.format(self.cluster_name)) hook = DataProcHook( gcp_conn_id=self.google_cloud_conn_id, delegate_to=self.delegate_to http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1cd6c4b0/tests/contrib/operators/test_dataproc_operator.py ---------------------------------------------------------------------- diff --git a/tests/contrib/operators/test_dataproc_operator.py b/tests/contrib/operators/test_dataproc_operator.py index 923fecc..a0c6ba0 100644 --- a/tests/contrib/operators/test_dataproc_operator.py +++ b/tests/contrib/operators/test_dataproc_operator.py @@ -13,14 +13,20 @@ # limitations under the License. # +import datetime import re import unittest +from airflow import DAG from airflow.contrib.operators.dataproc_operator import DataprocClusterCreateOperator +from airflow.contrib.operators.dataproc_operator import DataprocClusterDeleteOperator from airflow.version import version from copy import deepcopy +from mock import Mock +from mock import patch + TASK_ID = 'test-dataproc-operator' CLUSTER_NAME = 'test-cluster-name' PROJECT_ID = 'test-project-id' @@ -35,6 +41,7 @@ WORKER_DISK_SIZE = 100 NUM_PREEMPTIBLE_WORKERS = 2 LABEL1 = {} LABEL2 = {'application':'test', 'year': 2017} +DEFAULT_DATE = datetime.datetime(2017, 6, 6) class DataprocClusterCreateOperatorTest(unittest.TestCase): # Unitest for the DataprocClusterCreateOperator @@ -60,6 +67,14 @@ class DataprocClusterCreateOperatorTest(unittest.TestCase): labels = deepcopy(labels) ) ) + self.dag = DAG( + 'test_dag', + default_args={ + 'owner': 'airflow', + 'start_date': DEFAULT_DATE, + 'end_date': DEFAULT_DATE, + }, + schedule_interval='@daily') def test_init(self): """Test DataProcClusterOperator instance is properly initialized.""" @@ -95,3 +110,97 @@ class DataprocClusterCreateOperatorTest(unittest.TestCase): self.assertTrue(re.match(r'[a-z]([-a-z0-9]*[a-z0-9])?', cluster_data['labels']['airflow-version'])) self.assertEqual(cluster_data['labels'], merged_labels) + + def test_cluster_name_log_no_sub(self): + with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') \ + as mock_hook, patch('logging.info') as l: + dataproc_task = DataprocClusterCreateOperator( + task_id=TASK_ID, + cluster_name=CLUSTER_NAME, + project_id=PROJECT_ID, + num_workers=NUM_WORKERS, + zone=ZONE, + dag=self.dag + ) + + with self.assertRaises(TypeError) as _: + dataproc_task.execute(None) + l.assert_called_with(('Creating cluster: ' + CLUSTER_NAME)) + + def test_cluster_name_log_sub(self): + with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') \ + as mock_hook, patch('logging.info') as l: + dataproc_task = DataprocClusterCreateOperator( + task_id=TASK_ID, + cluster_name='smoke-cluster-{{ ts_nodash }}', + project_id=PROJECT_ID, + num_workers=NUM_WORKERS, + zone=ZONE, + dag=self.dag + ) + + context = { 'ts_nodash' : 'testnodash'} + + rendered = dataproc_task.render_template('cluster_name', getattr(dataproc_task,'cluster_name'), context) + setattr(dataproc_task, 'cluster_name', rendered) + with self.assertRaises(TypeError) as _: + dataproc_task.execute(None) + l.assert_called_with(('Creating cluster: smoke-cluster-testnodash')) + +class DataprocClusterDeleteOperatorTest(unittest.TestCase): + # Unitest for the DataprocClusterDeleteOperator + def setUp(self): + self.mock_execute = Mock() + self.mock_execute.execute = Mock(return_value={'done' : True}) + self.mock_get = Mock() + self.mock_get.get = Mock(return_value=self.mock_execute) + self.mock_operations = Mock() + self.mock_operations.get = Mock(return_value=self.mock_get) + self.mock_regions = Mock() + self.mock_regions.operations = Mock(return_value=self.mock_operations) + self.mock_projects=Mock() + self.mock_projects.regions = Mock(return_value=self.mock_regions) + self.mock_conn = Mock() + self.mock_conn.projects = Mock(return_value=self.mock_projects) + self.dag = DAG( + 'test_dag', + default_args={ + 'owner': 'airflow', + 'start_date': DEFAULT_DATE, + 'end_date': DEFAULT_DATE, + }, + schedule_interval='@daily') + + def test_cluster_name_log_no_sub(self): + with patch('airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook') \ + as mock_hook, patch('logging.info') as l: + mock_hook.return_value.get_conn = self.mock_conn + dataproc_task = DataprocClusterDeleteOperator( + task_id=TASK_ID, + cluster_name=CLUSTER_NAME, + project_id=PROJECT_ID, + dag=self.dag + ) + + with self.assertRaises(TypeError) as _: + dataproc_task.execute(None) + l.assert_called_with(('Deleting cluster: ' + CLUSTER_NAME)) + + def test_cluster_name_log_sub(self): + with patch('airflow.contrib.operators.dataproc_operator.DataProcHook') \ + as mock_hook, patch('logging.info') as l: + mock_hook.return_value.get_conn = self.mock_conn + dataproc_task = DataprocClusterDeleteOperator( + task_id=TASK_ID, + cluster_name='smoke-cluster-{{ ts_nodash }}', + project_id=PROJECT_ID, + dag=self.dag + ) + + context = { 'ts_nodash' : 'testnodash'} + + rendered = dataproc_task.render_template('cluster_name', getattr(dataproc_task,'cluster_name'), context) + setattr(dataproc_task, 'cluster_name', rendered) + with self.assertRaises(TypeError) as _: + dataproc_task.execute(None) + l.assert_called_with(('Deleting cluster: smoke-cluster-testnodash'))