From commits-return-18944-archive-asf-public=cust-asf.ponee.io@airflow.incubator.apache.org Thu Aug 23 11:40:02 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 5DF96180677 for ; Thu, 23 Aug 2018 11:40:02 +0200 (CEST) Received: (qmail 60989 invoked by uid 500); 23 Aug 2018 09:40:01 -0000 Mailing-List: contact commits-help@airflow.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@airflow.incubator.apache.org Delivered-To: mailing list commits@airflow.incubator.apache.org Received: (qmail 60980 invoked by uid 99); 23 Aug 2018 09:40:01 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 23 Aug 2018 09:40:01 +0000 From: GitBox To: commits@airflow.apache.org Subject: [GitHub] benjaminsims closed pull request #2417: [AIRFLOW-1380] Update import to work with latest Docker client Message-ID: <153501720093.21869.12314442689320009569.gitbox@gitbox.apache.org> Date: Thu, 23 Aug 2018 09:40:00 -0000 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit benjaminsims closed pull request #2417: [AIRFLOW-1380] Update import to work with latest Docker client URL: https://github.com/apache/incubator-airflow/pull/2417 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/airflow/hooks/docker_hook.py b/airflow/hooks/docker_hook.py index a570292ead..3512641c49 100644 --- a/airflow/hooks/docker_hook.py +++ b/airflow/hooks/docker_hook.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from docker import Client +from docker import APIClient as Client from docker.errors import APIError from airflow.exceptions import AirflowException diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py index 38edc8b4d7..fd540f1b8a 100644 --- a/airflow/operators/docker_operator.py +++ b/airflow/operators/docker_operator.py @@ -19,7 +19,7 @@ from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults from airflow.utils.file import TemporaryDirectory -from docker import Client, tls +from docker import APIClient as Client, tls import ast @@ -53,6 +53,9 @@ class DockerOperator(BaseOperator): :type environment: dict :param force_pull: Pull the docker image on every run. Default is false. :type force_pull: bool + :param host_tmp_dir: Specify the location of the temporary directory on the host which will + be mapped to tmp_dir + :type host_tmp_dir: str :param mem_limit: Maximum amount of memory the container can use. Either a float value, which represents the limit in bytes, or a string like ``128m`` or ``1g``. :type mem_limit: float or str @@ -101,6 +104,7 @@ def __init__( docker_url='unix://var/run/docker.sock', environment=None, force_pull=False, + host_tmp_dir=None, mem_limit=None, network_mode=None, tls_ca_cert=None, @@ -125,6 +129,7 @@ def __init__( self.docker_url = docker_url self.environment = environment or {} self.force_pull = force_pull + self.host_tmp_dir = host_tmp_dir self.image = image self.mem_limit = mem_limit self.network_mode = network_mode @@ -179,7 +184,7 @@ def execute(self, context): cpu_shares = int(round(self.cpus * 1024)) - with TemporaryDirectory(prefix='airflowtmp') as host_tmp_dir: + with TemporaryDirectory(prefix='airflowtmp', dir=self.host_tmp_dir) as host_tmp_dir: self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir self.volumes.append('{0}:{1}'.format(host_tmp_dir, self.tmp_dir)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..e0f58b87c5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +cryptography==2.0.3 +docker==2.5.1 +jsonschema==2.6.0 +tqdm==4.18.0 diff --git a/tests/operators/docker_operator.py b/tests/operators/docker_operator.py index a12b6f829f..4b83c1f22e 100644 --- a/tests/operators/docker_operator.py +++ b/tests/operators/docker_operator.py @@ -17,8 +17,8 @@ try: from airflow.operators.docker_operator import DockerOperator + from docker.client import APIClient as Client from airflow.hooks.docker_hook import DockerHook - from docker import Client except ImportError: pass @@ -51,6 +51,7 @@ def test_execute(self, client_class_mock, mkdtemp_mock): client_class_mock.return_value = client_mock operator = DockerOperator(api_version='1.19', command='env', environment={'UNIT': 'TEST'}, + host_tmp_dir='/host/airflow', image='ubuntu:latest', network_mode='bridge', owner='unittest', task_id='unittest', volumes=['/host/path:/container/path'], working_dir='/container/path') @@ -69,6 +70,8 @@ def test_execute(self, client_class_mock, mkdtemp_mock): mem_limit=None, user=None, working_dir='/container/path' ) + mkdtemp_mock.assert_called_with(dir='/host/airflow', + prefix='airflowtmp', suffix='') client_mock.create_host_config.assert_called_with(binds=['/host/path:/container/path', '/mkdtemp:/tmp/airflow'], network_mode='bridge') ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: users@infra.apache.org With regards, Apache Git Services