airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fo...@apache.org
Subject [19/20] incubator-airflow git commit: [AIRFLOW-1314] Polish some of the Kubernetes docs/config
Date Sun, 22 Apr 2018 08:32:52 GMT
[AIRFLOW-1314] Polish some of the Kubernetes docs/config


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/d807830f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/d807830f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/d807830f

Branch: refs/heads/master
Commit: d807830fe9f7f9232d7221b11a062b0787dc68eb
Parents: 317b6c7
Author: Fokko Driesprong <fokkodriesprong@godatadriven.com>
Authored: Wed Apr 18 13:21:04 2018 +0200
Committer: Fokko Driesprong <fokkodriesprong@godatadriven.com>
Committed: Sun Apr 22 10:23:06 2018 +0200

----------------------------------------------------------------------
 .gitignore                                      |   2 +
 scripts/ci/kubernetes/README.md                 |  16 ++
 scripts/ci/kubernetes/docker/airflow.tar.gz     | Bin 2374737 -> 0 bytes
 .../ci/kubernetes/kube/.generated/airflow.yaml  | 195 -------------------
 .../ci/kubernetes/kube/airflow.yaml.template    |   4 +-
 scripts/ci/kubernetes/kube/deploy.sh            |   4 +-
 6 files changed, 22 insertions(+), 199 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d807830f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 638ab19..04c408f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -136,3 +136,5 @@ rat-results.txt
 # Kubernetes generated templated files
 *.generated
 *.tar.gz
+scripts/ci/kubernetes/kube/.generated/airflow.yaml
+

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d807830f/scripts/ci/kubernetes/README.md
----------------------------------------------------------------------
diff --git a/scripts/ci/kubernetes/README.md b/scripts/ci/kubernetes/README.md
new file mode 100644
index 0000000..5d1f9c1
--- /dev/null
+++ b/scripts/ci/kubernetes/README.md
@@ -0,0 +1,16 @@
+# Airflow on Kubernetes
+
+If you don't have minikube installed, please run `./minikube/start_minikube.sh` to start
a k8s-instance on your local machine. Make sure that your `kubectl` is pointing to the local
k8s instance.
+
+First build the docker images by running `./docker/build.sh`. This will build the image and
push it to the local registry. Secondly, deploy Apache Airflow using `./kube/deploy.sh`. Finally,
open the Airflow webserver page by browsing to `http://192.168.99.100:30809/admin/` (on OSX).
+
+When kicking of a new job, you should be able to see new pods being kicked off:
+```
+$ kubectl get pods
+NAME                                                                  READY     STATUS  
           RESTARTS   AGE
+airflow-6cf894578b-rkcpm                                              2/2       Running 
           0          36m
+examplehttpoperatorhttpsensorcheck-490dc90941984812b934fceedf07ca81   1/1       Running 
           0          7s
+examplehttpoperatorhttpsensorcheck-ea787dd2163243a78dfba96d81f47e0d   1/1       Running 
           0          9s
+examplehttpoperatorpostop-3637d44e1b8a42789c59d2c6a66bec6a            0/1       ContainerCreating
  0          0s
+postgres-airflow-b4844754f-b8d8k                                      1/1       Running 
           0          36m
+```

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d807830f/scripts/ci/kubernetes/docker/airflow.tar.gz
----------------------------------------------------------------------
diff --git a/scripts/ci/kubernetes/docker/airflow.tar.gz b/scripts/ci/kubernetes/docker/airflow.tar.gz
deleted file mode 100644
index 114c5b4..0000000
Binary files a/scripts/ci/kubernetes/docker/airflow.tar.gz and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d807830f/scripts/ci/kubernetes/kube/.generated/airflow.yaml
----------------------------------------------------------------------
diff --git a/scripts/ci/kubernetes/kube/.generated/airflow.yaml b/scripts/ci/kubernetes/kube/.generated/airflow.yaml
deleted file mode 100644
index bd197d3..0000000
--- a/scripts/ci/kubernetes/kube/.generated/airflow.yaml
+++ /dev/null
@@ -1,195 +0,0 @@
-#  Licensed to the Apache Software Foundation (ASF) under one   *
-#  or more contributor license agreements.  See the NOTICE file *
-#  distributed with this work for additional information        *
-#  regarding copyright ownership.  The ASF licenses this file   *
-#  to you under the Apache License, Version 2.0 (the            *
-#  "License"); you may not use this file except in compliance   *
-#  with the License.  You may obtain a copy of the License at   *
-#                                                               *
-#    http://www.apache.org/licenses/LICENSE-2.0                 *
-#                                                               *
-#  Unless required by applicable law or agreed to in writing,   *
-#  software distributed under the License is distributed on an  *
-#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
-#  KIND, either express or implied.  See the License for the    *
-#  specific language governing permissions and limitations      *
-#  under the License.                                           *
-
-# The backing volume can be anything you want, it just needs to be `ReadWriteOnce`
-# I'm using hostPath since minikube is nice for testing, but any (non-local) volume will
work on a real cluster
-kind: PersistentVolume
-apiVersion: v1
-metadata:
-  name: airflow-dags
-  labels:
-    type: local
-spec:
-  capacity:
-    storage: 10Gi
-  accessModes:
-    - ReadWriteOnce
-  hostPath:
-    path: "/data/airflow-dags"
----
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-  name: airflow-dags
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Gi
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: airflow
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        name: airflow
-    spec:
-      initContainers:
-      - name: "init"
-        image: "airflow/ci:latest"
-        imagePullPolicy: "IfNotPresent"
-        volumeMounts:
-        - name: airflow-configmap
-          mountPath: /root/airflow/airflow.cfg
-          subPath: airflow.cfg
-        - name: airflow-dags
-          mountPath: /root/airflow/dags
-        env:
-        - name: SQL_ALCHEMY_CONN
-          valueFrom:
-            secretKeyRef:
-              name: airflow-secrets
-              key: sql_alchemy_conn
-        command:
-          - "bash"
-        args:
-          - "-cx"
-          - "cd /usr/local/lib/python2.7/dist-packages/airflow && cp -R example_dags/*
/root/airflow/dags/ && airflow initdb && alembic upgrade heads"
-      containers:
-      - name: web
-        image: airflow/ci:latest
-        imagePullPolicy: IfNotPresent
-        ports:
-        - name: web
-          containerPort: 8080
-        args: ["webserver"]
-        env:
-        - name: AIRFLOW_KUBE_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: SQL_ALCHEMY_CONN
-          valueFrom:
-            secretKeyRef:
-              name: airflow-secrets
-              key: sql_alchemy_conn
-        volumeMounts:
-        - name: airflow-configmap
-          mountPath: /root/airflow/airflow.cfg
-          subPath: airflow.cfg
-        - name: airflow-dags
-          mountPath: /root/airflow/dags
-        readinessProbe:
-          initialDelaySeconds: 5
-          timeoutSeconds: 5
-          periodSeconds: 5
-          httpGet:
-            path: /admin
-            port: 8080
-        livenessProbe:
-          initialDelaySeconds: 5
-          timeoutSeconds: 5
-          failureThreshold: 5
-          httpGet:
-            path: /admin
-            port: 8080
-      - name: scheduler
-        image: airflow/ci:latest
-        imagePullPolicy: IfNotPresent
-        args: ["scheduler"]
-        env:
-        - name: AIRFLOW_KUBE_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: SQL_ALCHEMY_CONN
-          valueFrom:
-            secretKeyRef:
-              name: airflow-secrets
-              key: sql_alchemy_conn
-        volumeMounts:
-        - name: airflow-configmap
-          mountPath: /root/airflow/airflow.cfg
-          subPath: airflow.cfg
-        - name: airflow-dags
-          mountPath: /root/airflow/dags
-      volumes:
-      - name: airflow-dags
-        persistentVolumeClaim:
-          claimName: airflow-dags
-      - name: airflow-configmap
-        configMap:
-          name: airflow-configmap
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: airflow
-spec:
-  type: NodePort
-  ports:
-    - port: 8080
-      nodePort: 30809
-  selector:
-    name: airflow
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: airflow-secrets
-type: Opaque
-data:
-  # The sql_alchemy_conn value is a base64 encoded represenation of this connection string:
-  # postgresql+psycopg2://root:root@postgres-airflow:5432/airflow
-  sql_alchemy_conn: cG9zdGdyZXNxbCtwc3ljb3BnMjovL3Jvb3Q6cm9vdEBwb3N0Z3Jlcy1haXJmbG93OjU0MzIvYWlyZmxvdwo=
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: airflow-configmap
-data:
-  airflow.cfg: |
-    [core]
-    airflow_home = /root/airflow
-    dags_folder = /root/airflow/dags
-    base_log_folder = /root/airflow/logs
-    logging_level = INFO
-    executor = KubernetesExecutor
-    parallelism = 32
-    plugins_folder = /root/airflow/plugins
-    sql_alchemy_conn = $SQL_ALCHEMY_CONN
-
-    [scheduler]
-    dag_dir_list_interval = 60
-    child_process_log_directory = /root/airflow/logs/scheduler
-
-    [kubernetes]
-    airflow_configmap = airflow-configmap
-    worker_container_repository = airflow/ci
-    worker_container_tag = latest
-    delete_worker_pods = False
-    git_repo = https://github.com/grantnicholas/testdags.git
-    git_branch = master
-    dags_volume_claim = airflow-dags
-
-    [kubernetes_secrets]
-    SQL_ALCHEMY_CONN = airflow-secrets=sql_alchemy_conn

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d807830f/scripts/ci/kubernetes/kube/airflow.yaml.template
----------------------------------------------------------------------
diff --git a/scripts/ci/kubernetes/kube/airflow.yaml.template b/scripts/ci/kubernetes/kube/airflow.yaml.template
index e16cc08..3f791a1 100644
--- a/scripts/ci/kubernetes/kube/airflow.yaml.template
+++ b/scripts/ci/kubernetes/kube/airflow.yaml.template
@@ -90,11 +90,11 @@ spec:
           - "-cx"
           - "cd /usr/local/lib/python2.7/dist-packages/airflow && cp -R example_dags/*
/root/airflow/dags/ && airflow initdb && alembic upgrade heads"
       containers:
-      - name: web
+      - name: webserver
         image: {{docker_image}}:{{docker_tag}}
         imagePullPolicy: IfNotPresent
         ports:
-        - name: web
+        - name: webserver
           containerPort: 8080
         args: ["webserver"]
         env:

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d807830f/scripts/ci/kubernetes/kube/deploy.sh
----------------------------------------------------------------------
diff --git a/scripts/ci/kubernetes/kube/deploy.sh b/scripts/ci/kubernetes/kube/deploy.sh
index 27c707f..32bed83 100755
--- a/scripts/ci/kubernetes/kube/deploy.sh
+++ b/scripts/ci/kubernetes/kube/deploy.sh
@@ -44,8 +44,8 @@ POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.na
 
 echo "------- pod description -------"
 kubectl describe pod $POD
-echo "------- web logs -------"
-kubectl logs $POD web
+echo "------- webserver logs -------"
+kubectl logs $POD webserver
 echo "------- scheduler logs -------"
 kubectl logs $POD scheduler
 echo "--------------"


Mime
View raw message