airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (AIRFLOW-3262) Can't get log containing Response when using SimpleHttpOperator
Date Fri, 02 Nov 2018 13:34:00 GMT

    [ https://issues.apache.org/jira/browse/AIRFLOW-3262?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16673100#comment-16673100
] 

ASF GitHub Bot commented on AIRFLOW-3262:
-----------------------------------------

kaxil closed pull request #4102: [AIRFLOW-3262] Add param to log response when using SimpleHttpOperator
URL: https://github.com/apache/incubator-airflow/pull/4102
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/airflow/operators/http_operator.py b/airflow/operators/http_operator.py
index 0585a92a86..3e00de96eb 100644
--- a/airflow/operators/http_operator.py
+++ b/airflow/operators/http_operator.py
@@ -46,6 +46,10 @@ class SimpleHttpOperator(BaseOperator):
         'requests' documentation (options to modify timeout, ssl, etc.)
     :type extra_options: A dictionary of options, where key is string and value
         depends on the option that's being modified.
+    :param xcom_push: Push the response to Xcom (default: False)
+    :type xcom_push: bool
+    :param log_response: Log the response (default: False)
+    :type log_response: bool
     """
 
     template_fields = ('endpoint', 'data',)
@@ -61,7 +65,9 @@ def __init__(self,
                  response_check=None,
                  extra_options=None,
                  xcom_push=False,
-                 http_conn_id='http_default', *args, **kwargs):
+                 http_conn_id='http_default',
+                 log_response=False,
+                 *args, **kwargs):
         """
         If xcom_push is True, response of an HTTP request will also
         be pushed to an XCom.
@@ -75,6 +81,7 @@ def __init__(self,
         self.response_check = response_check
         self.extra_options = extra_options or {}
         self.xcom_push_flag = xcom_push
+        self.log_response = log_response
 
     def execute(self, context):
         http = HttpHook(self.method, http_conn_id=self.http_conn_id)
@@ -90,3 +97,5 @@ def execute(self, context):
                 raise AirflowException("Response check returned False.")
         if self.xcom_push_flag:
             return response.text
+        if self.log_response:
+            self.log.info(response.text)
diff --git a/tests/operators/test_http_operator.py b/tests/operators/test_http_operator.py
new file mode 100644
index 0000000000..6ab2c03bb5
--- /dev/null
+++ b/tests/operators/test_http_operator.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import unittest
+
+from mock import patch
+from airflow.operators.http_operator import SimpleHttpOperator
+
+try:
+    from unittest import mock
+except ImportError:
+    try:
+        import mock
+    except ImportError:
+        mock = None
+
+
+class AnyStringWith(str):
+    """
+    Helper class to check if a substring is a part of a string
+    """
+    def __eq__(self, other):
+        return self in other
+
+
+class SimpleHttpOpTests(unittest.TestCase):
+    def setUp(self):
+        # Creating a local Http connection to Airflow Webserver
+        os.environ['AIRFLOW_CONN_HTTP_GOOGLE'] = 'http://www.google.com'
+
+    def test_response_in_logs(self):
+        """
+        Test that when using SimpleHttpOperator with 'GET' on localhost:8080,
+        the log contains 'Google' in it
+        """
+        operator = SimpleHttpOperator(
+            task_id='test_HTTP_op',
+            method='GET',
+            endpoint='/',
+            http_conn_id='HTTP_GOOGLE',
+            log_response=True,
+        )
+
+        with patch.object(operator.log, 'info') as mock_info:
+            operator.execute(None)
+            mock_info.assert_called_with(AnyStringWith('Google'))


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> Can't get log containing Response when using SimpleHttpOperator
> ---------------------------------------------------------------
>
>                 Key: AIRFLOW-3262
>                 URL: https://issues.apache.org/jira/browse/AIRFLOW-3262
>             Project: Apache Airflow
>          Issue Type: Improvement
>            Reporter: Kaxil Naik
>            Assignee: Kaxil Naik
>            Priority: Trivial
>             Fix For: 1.10.1
>
>
> When you use SimpleHttpOperator for things like ElasticSearch, you want to get the response
in the logs as well. Currently, the only workaround is to use `xcom_push` and push the content
to xcom and in the next task get the response.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message