airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-40] Add LDAP group filtering feature.
Date Tue, 28 Jun 2016 19:12:52 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master 02a2076a7 -> d6d3f5367


[AIRFLOW-40] Add LDAP group filtering feature.

It is now possible to filter over LDAP group (in the web
interface) when using the LDAP authentication backend.
Note that this feature requires the "memberOf"
overlay to be configured on the LDAP server.

Closes #1479 from dsjl/AIRFLOW-40


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/d6d3f536
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/d6d3f536
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/d6d3f536

Branch: refs/heads/master
Commit: d6d3f53673ba3736d7a858531823933cfef2bb4e
Parents: 02a2076
Author: Damien Lejeune <damien.lejeune@klarna.com>
Authored: Tue Jun 28 21:12:44 2016 +0200
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Tue Jun 28 21:12:44 2016 +0200

----------------------------------------------------------------------
 airflow/configuration.py                   | 24 ++++++++++++
 airflow/contrib/auth/backends/ldap_auth.py | 39 +++++++++++++++++++
 airflow/www/views.py                       | 50 ++++++++++++++++++-------
 scripts/ci/ldif/groups.example.com.ldif    | 39 +++++++++++++++++++
 scripts/ci/ldif/users.example.com.ldif     |  9 +++++
 scripts/ci/load_fixtures.sh                |  2 +-
 scripts/ci/slapd.conf                      |  5 ++-
 tests/core.py                              | 33 ++++++++++++++++
 8 files changed, 185 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d6d3f536/airflow/configuration.py
----------------------------------------------------------------------
diff --git a/airflow/configuration.py b/airflow/configuration.py
index e92ba8a..c0e574b 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -118,6 +118,7 @@ defaults = {
         'web_server_worker_timeout': 120,
         'authenticate': False,
         'filter_by_owner': False,
+        'owner_mode': 'user',
         'demo_mode': False,
         'secret_key': 'airflowified',
         'expose_config': False,
@@ -294,6 +295,13 @@ authenticate = False
 # Filter the list of dags by owner name (requires authentication to be enabled)
 filter_by_owner = False
 
+# Filtering mode. Choices include user (default) and ldapgroup.
+# Ldap group filtering requires using the ldap backend
+#
+# Note that the ldap server needs the "memberOf" overlay to be set up
+# in order to user the ldapgroup mode.
+owner_mode = user
+
 [email]
 email_backend = airflow.utils.email.send_email_smtp
 
@@ -487,6 +495,22 @@ class ConfigParserWithDefaults(ConfigParser):
             raise AirflowConfigException("error: cannot use sqlite with the {}".
                 format(self.get('core', 'executor')))
 
+        elif (
+            self.getboolean("webserver", "authenticate") and
+            self.get("webserver", "owner_mode") not in ['user', 'ldapgroup']
+        ):
+            raise AirflowConfigException("error: owner_mode option should be either "
+                                         "'user' or 'ldapgroup' "
+                                         "when filtering by owner is set")
+
+        elif (
+            self.getboolean("webserver", "authenticate") and
+            self.get("webserver", "owner_mode").lower() == 'ldapgroup' and
+            self.get("core", "auth_backend") != 'airflow.contrib.auth.backends.ldap_auth'
+        ):
+            raise AirflowConfigException("error: attempt at using ldapgroup "
+                                         "filtering without using the Ldap backend")
+
         self.is_validated = True
 
     def _get_env_var_option(self, section, key):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d6d3f536/airflow/contrib/auth/backends/ldap_auth.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/auth/backends/ldap_auth.py b/airflow/contrib/auth/backends/ldap_auth.py
index 7ea6d87..f79529e 100644
--- a/airflow/contrib/auth/backends/ldap_auth.py
+++ b/airflow/contrib/auth/backends/ldap_auth.py
@@ -32,6 +32,7 @@ from airflow.configuration import AirflowConfigException
 import logging
 
 import traceback
+import re
 
 login_manager = flask_login.LoginManager()
 login_manager.login_view = 'airflow.login'  # Calls login() bellow
@@ -78,9 +79,37 @@ def group_contains_user(conn, search_base, group_filter, user_name_attr,
usernam
     return False
 
 
+def groups_user(conn, search_base, user_filter, user_name_att, username):
+    search_filter = "(&({0})({1}={2}))".format(user_filter, user_name_att, username)
+    res = conn.search(search_base, search_filter, attributes=["memberOf"])
+    if not res:
+        LOG.info("Cannot find user %s", username)
+        raise AuthenticationError("Invalid username or password")
+
+    if conn.response and "memberOf" not in conn.response[0]["attributes"]:
+        LOG.warn("""Missing attribute "memberOf" when looked-up in Ldap database.
+        The user does not seem to be a member of a group and therefore won't see any dag
+        if the option filter_by_owner=True and owner_mode=ldapgroup are set""")
+        return []
+
+    user_groups = conn.response[0]["attributes"]["memberOf"]
+
+    regex = re.compile("cn=([^,]*).*")
+    groups_list = []
+    try:
+        groups_list = [regex.search(i).group(1) for i in user_groups]
+    except IndexError:
+        LOG.warning("Parsing error when retrieving the user's group(s)."
+                    " Check if the user belongs to at least one group"
+                    " or if the user's groups name do not contain special characters")
+
+    return groups_list
+
+
 class LdapUser(models.User):
     def __init__(self, user):
         self.user = user
+        self.ldap_groups = []
 
         # Load and cache superuser and data_profiler settings.
         conn = get_ldap_connection(configuration.get("ldap", "bind_user"), configuration.get("ldap",
"bind_password"))
@@ -104,6 +133,16 @@ class LdapUser(models.User):
             self.data_profiler = True
             LOG.debug("Missing configuration for dataprofiler settings. Skipping")
 
+        # Load the ldap group(s) a user belongs to
+        try:
+            self.ldap_groups = groups_user(conn,
+                                           configuration.get("ldap", "basedn"),
+                                           configuration.get("ldap", "user_filter"),
+                                           configuration.get("ldap", "user_name_attr"),
+                                           user.username)
+        except AirflowConfigException:
+            LOG.debug("Missing configuration for ldap settings. Skipping")
+
     @staticmethod
     def try_login(username, password):
         conn = get_ldap_connection(configuration.get("ldap", "bind_user"), configuration.get("ldap",
"bind_password"))

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d6d3f536/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 1fb3f91..188bd79 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -1619,17 +1619,30 @@ class HomeView(AdminIndexView):
         qry = None
         # filter the dags if filter_by_owner and current user is not superuser
         do_filter = FILTER_BY_OWNER and (not current_user.is_superuser())
+        owner_mode = conf.get('webserver', 'OWNER_MODE').strip().lower()
+
+        qry = session.query(DM)
+        qry_fltr = []
+
         if do_filter:
-            qry = (
-                session.query(DM)
-                    .filter(
-                    ~DM.is_subdag, DM.is_active,
-                    DM.owners.like('%' + current_user.username + '%'))
+            if owner_mode == 'ldapgroup':
+                qry_fltr = (
+                    qry.filter(
+                        ~DM.is_subdag, DM.is_active,
+                        DM.owners.in_(current_user.ldap_groups))
                     .all()
-            )
+                )
+            elif owner_mode == 'user':
+                qry_fltr = (
+                    qry.filter(
+                        ~DM.is_subdag, DM.is_active,
+                        DM.owners == current_user.user.username)
+                    .all()
+                )
         else:
-            qry = session.query(DM).filter(~DM.is_subdag, DM.is_active).all()
-        orm_dags = {dag.dag_id: dag for dag in qry}
+            qry_fltr = qry.filter(~DM.is_subdag, DM.is_active).all()
+
+        orm_dags = {dag.dag_id: dag for dag in qry_fltr}
         import_errors = session.query(models.ImportError).all()
         for ie in import_errors:
             flash(
@@ -1640,12 +1653,21 @@ class HomeView(AdminIndexView):
         session.close()
         dags = dagbag.dags.values()
         if do_filter:
-            dags = {
-                dag.dag_id: dag
-                for dag in dags
-                if (
-                    dag.owner == current_user.username and (not dag.parent_dag)
-                )
+            if owner_mode == 'ldapgroup':
+                dags = {
+                    dag.dag_id: dag
+                    for dag in dags
+                    if (
+                        dag.owner in current_user.ldap_groups and (not dag.parent_dag)
+                    )
+                }
+            elif owner_mode == 'user':
+                dags = {
+                    dag.dag_id: dag
+                    for dag in dags
+                    if (
+                        dag.owner == current_user.user.username and (not dag.parent_dag)
+                    )
                 }
         else:
             dags = {dag.dag_id: dag for dag in dags if not dag.parent_dag}

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d6d3f536/scripts/ci/ldif/groups.example.com.ldif
----------------------------------------------------------------------
diff --git a/scripts/ci/ldif/groups.example.com.ldif b/scripts/ci/ldif/groups.example.com.ldif
new file mode 100644
index 0000000..21ddb89
--- /dev/null
+++ b/scripts/ci/ldif/groups.example.com.ldif
@@ -0,0 +1,39 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# create all groups
+
+dn: ou=groups,dc=example,dc=com
+objectClass: organizationalunit
+ou: groups
+description: generic groups branch
+
+dn: cn=group1,ou=groups,dc=example,dc=com
+objectclass: groupofnames
+cn: group1
+description: Group 1 of users
+# add the group members all of which are 
+# assumed to exist under example
+member: cn=user1,dc=example,dc=com
+
+dn: cn=group2,ou=groups,dc=example,dc=com
+objectclass: groupofnames
+cn: group2
+description: Group 2 of users
+member: cn=user2,dc=example,dc=com
+
+dn: cn=group3,ou=groups,dc=example,dc=com
+objectclass: groupofnames
+cn: group3
+description: Group 3 of users
+member: cn=user1,dc=example,dc=com

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d6d3f536/scripts/ci/ldif/users.example.com.ldif
----------------------------------------------------------------------
diff --git a/scripts/ci/ldif/users.example.com.ldif b/scripts/ci/ldif/users.example.com.ldif
index 0f35839..3cec13a 100644
--- a/scripts/ci/ldif/users.example.com.ldif
+++ b/scripts/ci/ldif/users.example.com.ldif
@@ -18,6 +18,15 @@ objectClass: account
 objectClass: simpleSecurityObject
 uid: user1
 userPassword: user1
+memberOf: cn=group1,dc=example,dc=com
+memberOf: cn=group3,dc=example,dc=com
+
+dn: uid=user2,dc=example,dc=com
+objectClass: account
+objectClass: simpleSecurityObject
+uid: user2
+userPassword: user2
+memberOf: cn=group2,dc=example,dc=com
 
 dn: uid=dataprofiler,dc=example,dc=com
 objectClass: account

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d6d3f536/scripts/ci/load_fixtures.sh
----------------------------------------------------------------------
diff --git a/scripts/ci/load_fixtures.sh b/scripts/ci/load_fixtures.sh
index 0aa92cc..259fe82 100755
--- a/scripts/ci/load_fixtures.sh
+++ b/scripts/ci/load_fixtures.sh
@@ -17,7 +17,7 @@ set -o verbose
 
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 FIXTURES_DIR="$DIR/ldif"
-LOAD_ORDER=("example.com.ldif" "manager.example.com.ldif" "users.example.com.ldif")
+LOAD_ORDER=("example.com.ldif" "manager.example.com.ldif" "users.example.com.ldif" "groups.example.com.ldif")
 
 load_fixture () {
   ldapadd -x -H ldap://127.0.0.1:3890/ -D "cn=Manager,dc=example,dc=com" -w insecure -f $1

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d6d3f536/scripts/ci/slapd.conf
----------------------------------------------------------------------
diff --git a/scripts/ci/slapd.conf b/scripts/ci/slapd.conf
index e0ec94c..302c355 100644
--- a/scripts/ci/slapd.conf
+++ b/scripts/ci/slapd.conf
@@ -22,6 +22,7 @@ include         /etc/ldap/schema/nis.schema
 include         /etc/ldap/schema/inetorgperson.schema
 
 moduleload back_hdb
+moduleload memberof.la
 
 disallow bind_anon
 
@@ -54,4 +55,6 @@ access to attrs=userPassword
   by anonymous auth
   by users none
 
-access to * by * read
\ No newline at end of file
+access to * by * read
+
+overlay memberof

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d6d3f536/tests/core.py
----------------------------------------------------------------------
diff --git a/tests/core.py b/tests/core.py
index 9c688ea..0e37848 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -1173,6 +1173,39 @@ class WebLdapAuthTest(unittest.TestCase):
         session.close()
         configuration.conf.set("webserver", "authenticate", "False")
 
+
+class LdapGroupTest(unittest.TestCase):
+    def setUp(self):
+        configuration.conf.set("webserver", "authenticate", "True")
+        configuration.conf.set("webserver", "auth_backend", "airflow.contrib.auth.backends.ldap_auth")
+        try:
+            configuration.conf.add_section("ldap")
+        except:
+            pass
+        configuration.conf.set("ldap", "uri", "ldap://localhost:3890")
+        configuration.conf.set("ldap", "user_filter", "objectClass=*")
+        configuration.conf.set("ldap", "user_name_attr", "uid")
+        configuration.conf.set("ldap", "bind_user", "cn=Manager,dc=example,dc=com")
+        configuration.conf.set("ldap", "bind_password", "insecure")
+        configuration.conf.set("ldap", "basedn", "dc=example,dc=com")
+        configuration.conf.set("ldap", "cacert", "")
+
+    def test_group_belonging(self):
+        from airflow.contrib.auth.backends.ldap_auth import LdapUser
+        users = {"user1": ["group1", "group3"],
+                 "user2": ["group2"]
+                 }
+        for user in users:
+            mu = models.User(username=user,
+                             is_superuser=False)
+            auth = LdapUser(mu)
+            assert set(auth.ldap_groups) == set(users[user])
+
+    def tearDown(self):
+        configuration.test_mode()
+        configuration.conf.set("webserver", "authenticate", "False")
+
+
 class FakeSession(object):
     def __init__(self):
         from requests import Response


Mime
View raw message