airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From machris...@apache.org
Subject [airavata] 02/02: AIRAVATA-2775 Load roles from Keycloak to create gateway groups
Date Thu, 10 May 2018 16:28:53 GMT
This is an automated email from the ASF dual-hosted git repository.

machristie pushed a commit to branch group-based-auth
in repository https://gitbox.apache.org/repos/asf/airavata.git

commit 4f1bced37cd0cec44acefb4257066289eebde912
Author: Marcus Christie <machristie@apache.org>
AuthorDate: Thu May 10 12:23:03 2018 -0400

    AIRAVATA-2775 Load roles from Keycloak to create gateway groups
    
    Also, simplified, documented how to run AiravataDataMigrator
    (AIRAVATA-2704).
---
 .../sharing-data-migrator/README.md                |   7 +
 .../sharing-registry/sharing-data-migrator/pom.xml |  11 ++
 .../migrator/airavata/AiravataDataMigrator.java    | 154 ++++++++++++++++-----
 .../migrator/airavata/ConnectionFactory.java       |  23 ++-
 .../src/main/resources/.gitignore                  |   1 +
 .../resources/airavata-server.properties.sample    |  66 +++++++++
 .../registry/utils/ThriftDataModelConversion.java  |   1 +
 7 files changed, 221 insertions(+), 42 deletions(-)

diff --git a/modules/sharing-registry/sharing-data-migrator/README.md b/modules/sharing-registry/sharing-data-migrator/README.md
new file mode 100644
index 0000000..03d452a
--- /dev/null
+++ b/modules/sharing-registry/sharing-data-migrator/README.md
@@ -0,0 +1,7 @@
+
+# Running the Airavata Data Migrator
+
+1. Copy the file `src/main/resources/airavata-server.properties.sample` to `src/main/resources/airavata-server.properties`.
+2. Edit the `airavata-server.properties` file and make sure all properties are specified
correctly.
+3. Make sure that you have run `mvn install` in the root directory of the airavata project.
+4. Run the `AiravataDataMigrator` class.
\ No newline at end of file
diff --git a/modules/sharing-registry/sharing-data-migrator/pom.xml b/modules/sharing-registry/sharing-data-migrator/pom.xml
index 8d7bae6..b2dead9 100644
--- a/modules/sharing-registry/sharing-data-migrator/pom.xml
+++ b/modules/sharing-registry/sharing-data-migrator/pom.xml
@@ -73,6 +73,12 @@
             <groupId>org.apache.airavata</groupId>
             <artifactId>services-security</artifactId>
             <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents.wso2</groupId>
+                    <artifactId>httpcore</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.thrift</groupId>
@@ -85,6 +91,11 @@
             <version>0.17-SNAPSHOT</version>
         </dependency>
         <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>iam-admin-services-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
             <groupId>log4j</groupId>
             <artifactId>log4j</artifactId>
             <version>1.2.17</version>
diff --git a/modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/AiravataDataMigrator.java
b/modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/AiravataDataMigrator.java
index cce13c1..677e5d1 100644
--- a/modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/AiravataDataMigrator.java
+++ b/modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/AiravataDataMigrator.java
@@ -20,27 +20,42 @@
 package org.apache.airavata.sharing.registry.migrator.airavata;
 
 import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
-import org.apache.airavata.model.group.ResourcePermissionType;
-import org.apache.airavata.model.group.ResourceType;
-import org.apache.airavata.sharing.registry.models.*;
-import org.apache.airavata.sharing.registry.server.SharingRegistryServerHandler;
-import org.apache.thrift.TException;
-import org.apache.airavata.credential.store.cpi.CredentialStoreService;
+import org.apache.airavata.common.utils.AiravataUtils;
 import org.apache.airavata.common.utils.ServerSettings;
 import org.apache.airavata.credential.store.client.CredentialStoreClientFactory;
+import org.apache.airavata.credential.store.cpi.CredentialStoreService;
 import org.apache.airavata.credential.store.exception.CredentialStoreException;
+import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
 import org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile;
 import org.apache.airavata.model.credential.store.PasswordCredential;
+import org.apache.airavata.model.group.ResourcePermissionType;
+import org.apache.airavata.model.group.ResourceType;
+import org.apache.airavata.registry.api.RegistryService;
 import org.apache.airavata.registry.api.client.RegistryServiceClientFactory;
 import org.apache.airavata.registry.api.exception.RegistryServiceException;
-import org.apache.airavata.registry.api.RegistryService;
+import org.apache.airavata.service.profile.iam.admin.services.core.impl.TenantManagementKeycloakImpl;
+import org.apache.airavata.service.profile.iam.admin.services.cpi.exception.IamAdminServicesException;
+import org.apache.airavata.sharing.registry.models.Domain;
+import org.apache.airavata.sharing.registry.models.Entity;
+import org.apache.airavata.sharing.registry.models.EntityType;
+import org.apache.airavata.sharing.registry.models.GroupCardinality;
+import org.apache.airavata.sharing.registry.models.GroupType;
+import org.apache.airavata.sharing.registry.models.PermissionType;
+import org.apache.airavata.sharing.registry.models.User;
+import org.apache.airavata.sharing.registry.models.UserGroup;
+import org.apache.airavata.sharing.registry.server.SharingRegistryServerHandler;
+import org.apache.thrift.TException;
 
-import java.util.*;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class AiravataDataMigrator {
 
@@ -219,19 +234,20 @@ public class AiravataDataMigrator {
 
         //Map to reuse the domain ID and owner for creating application-deployment entities
         Map<String, String> domainOwnerMap = new HashMap<>();
+        Map<String, Map<String, String>> gatewayGroupsMap = new HashMap<>();
 
-        //Creating the everyone group
+        //Creating the gateway groups
         List<Domain> domainList = sharingRegistryServerHandler.getDomains(0, -1);
         for (Domain domain : domainList) {
             GatewayResourceProfile gatewayResourceProfile = null;
             try {
                 gatewayResourceProfile = getRegistryServiceClient().getGatewayResourceProfile(domain.domainId);
             } catch (Exception e) {
-                System.out.println("Skipping creating everyone group for " + domain.domainId
+ " because it doesn't have a GatewayResourceProfile");
+                System.out.println("Skipping creating groups for " + domain.domainId + "
because it doesn't have a GatewayResourceProfile");
                 continue;
             }
             if (gatewayResourceProfile.getIdentityServerPwdCredToken() == null) {
-                System.out.println("Skipping creating everyone group for " + domain.domainId
+ " because it doesn't have an identity server pwd credential token");
+                System.out.println("Skipping creating groups for " + domain.domainId + "
because it doesn't have an identity server pwd credential token");
                 continue;
             }
             String groupOwner = null;
@@ -240,37 +256,47 @@ public class AiravataDataMigrator {
                         gatewayResourceProfile.getIdentityServerPwdCredToken(), gatewayResourceProfile.getGatewayID());
                 groupOwner = credential.getLoginUserName();
             } catch (Exception e) {
-                System.out.println("Skipping creating everyone group for " + domain.domainId
+ " because the identity server pwd credential could not be retrieved.");
+                System.out.println("Skipping creating groups for " + domain.domainId + "
because the identity server pwd credential could not be retrieved.");
                 continue;
             }
 
             domainOwnerMap.put(domain.domainId, groupOwner);
-            String groupId = "everyone@" + domain.domainId;
+
             String ownerId = groupOwner + "@" + domain.domainId;
             if (!sharingRegistryServerHandler.isUserExists(domain.domainId, ownerId)) {
-                System.out.println("Skipping creating everyone group for " + domain.domainId
+ " because admin user doesn't exist in sharing registry.");
+                System.out.println("Skipping creating groups for " + domain.domainId + "
because admin user doesn't exist in sharing registry.");
                 continue;
             }
-            if (!sharingRegistryServerHandler.isGroupExists(domain.domainId, groupId)) {
-                UserGroup userGroup = new UserGroup();
-                userGroup.setGroupId(groupId);
-                userGroup.setDomainId(domain.domainId);
-                userGroup.setGroupCardinality(GroupCardinality.MULTI_USER);
-                userGroup.setCreatedTime(System.currentTimeMillis());
-                userGroup.setUpdatedTime(System.currentTimeMillis());
-                userGroup.setName("everyone");
-                userGroup.setDescription("Default Group");
-                userGroup.setOwnerId(ownerId);
-                userGroup.setGroupType(GroupType.DOMAIN_LEVEL_GROUP);
-                sharingRegistryServerHandler.createGroup(userGroup);
-
-                List<User> userList = sharingRegistryServerHandler.getUsers(domain.domainId,
0, -1);
-                List<String> users = new ArrayList<>();
-                for (User user : userList) {
-                    users.add(user.getUserId());
-                }
-                sharingRegistryServerHandler.addUsersToGroup(domain.domainId, users, groupId);
+
+            List<String> usernames = sharingRegistryServerHandler.getUsers(domain.domainId,
0, -1)
+                    .stream()
+                    // Filter out bad ids that don't have an "@" in them
+                    .filter(user -> user.getUserId().lastIndexOf("@") > 0)
+                    .map(user -> user.getUserId().substring(0, user.getUserId().lastIndexOf("@")))
+                    .collect(Collectors.toList());
+            Map<String,List<String>> roleMap = loadRolesForUsers(domain.domainId,
usernames);
+
+            Map<String, String> gatewayGroupIds = new HashMap<>();
+            if (roleMap.containsKey("gateway-user")) {
+                UserGroup gatewayUsersGroup = createGroup(sharingRegistryServerHandler, domain,
ownerId,
+                        "Gateway Users",
+                        "Default group for users of the gateway.", roleMap.get("gateway-user"));
+                gatewayGroupIds.put("gateway-users", gatewayUsersGroup.groupId);
             }
+            if (roleMap.containsKey("admin")) {
+                UserGroup adminUsersGroup = createGroup(sharingRegistryServerHandler, domain,
ownerId,
+                        "Admin Users",
+                        "Admin users group.", roleMap.get("admin"));
+                gatewayGroupIds.put("admins", adminUsersGroup.groupId);
+            }
+            if (roleMap.containsKey("admin-read-only")) {
+                UserGroup adminUsersGroup = createGroup(sharingRegistryServerHandler, domain,
ownerId,
+                        "Read Only Admin Users",
+                        "Group of admin users with read-only access.", roleMap.get("admin-read-only"));
+                gatewayGroupIds.put("read-only-admins", adminUsersGroup.groupId);
+            }
+            gatewayGroupsMap.put(domain.domainId, gatewayGroupIds);
+
         }
 
         //Creating application deployment entries
@@ -291,9 +317,9 @@ public class AiravataDataMigrator {
 
                 if (!sharingRegistryServerHandler.isEntityExists(entity.domainId, entity.entityId))
                     sharingRegistryServerHandler.createEntity(entity);
-                String groupID = "everyone@" + entity.domainId;
+                String groupID = gatewayGroupsMap.get(domainID).get("gateway-users");
                 sharingRegistryServerHandler.shareEntityWithGroups(entity.domainId, entity.entityId,
Arrays.asList(groupID),
-                        entity.domainId + ":" + ResourcePermissionType.EXEC, true);
+                        entity.domainId + ":" + ResourcePermissionType.READ, true);
             }
         }
 
@@ -302,6 +328,62 @@ public class AiravataDataMigrator {
 
     }
 
+    private static Map<String,List<String>> loadRolesForUsers(String gatewayId,
List<String> usernames) throws TException, ApplicationSettingsException {
+
+        TenantManagementKeycloakImpl tenantManagementKeycloak = new TenantManagementKeycloakImpl();
+        PasswordCredential tenantAdminPasswordCredential = getTenantAdminPasswordCredential(gatewayId);
+        Map<String, List<String>> roleMap = new HashMap<>();
+        for (String username : usernames) {
+            try {
+                List<String> roles = tenantManagementKeycloak.getUserRoles(tenantAdminPasswordCredential,
gatewayId, username);
+                if (roles != null) {
+                    for (String role : roles) {
+                        if (!roleMap.containsKey(role)) {
+                            roleMap.put(role, new ArrayList<>());
+                        }
+                        roleMap.get(role).add(username);
+                    }
+                } else {
+                    System.err.println("Warning: user [" + username + "] in tenant [" + gatewayId
+ "] has no roles.");
+                }
+            } catch (IamAdminServicesException e) {
+                System.err.println("Error: unable to load roles for user [" + username +
"] in tenant [" + gatewayId + "].");
+                e.printStackTrace();
+            }
+        }
+        return roleMap;
+    }
+
+    private static PasswordCredential getTenantAdminPasswordCredential(String tenantId) throws
TException, ApplicationSettingsException {
+
+        GatewayResourceProfile gwrp = getRegistryServiceClient().getGatewayResourceProfile(tenantId);
+
+        CredentialStoreService.Client csClient = getCredentialStoreServiceClient();
+        return csClient.getPasswordCredential(gwrp.getIdentityServerPwdCredToken(), gwrp.getGatewayID());
+    }
+
+    private static UserGroup createGroup(SharingRegistryServerHandler sharingRegistryServerHandler,
Domain domain, String ownerId, String groupName, String groupDescription, List<String>
usernames) throws TException {
+
+        UserGroup userGroup = new UserGroup();
+        userGroup.setGroupId(AiravataUtils.getId(groupName));
+        userGroup.setDomainId(domain.domainId);
+        userGroup.setGroupCardinality(GroupCardinality.MULTI_USER);
+        userGroup.setCreatedTime(System.currentTimeMillis());
+        userGroup.setUpdatedTime(System.currentTimeMillis());
+        userGroup.setName(groupName);
+        userGroup.setDescription(groupDescription);
+        userGroup.setOwnerId(ownerId);
+        userGroup.setGroupType(GroupType.DOMAIN_LEVEL_GROUP);
+        sharingRegistryServerHandler.createGroup(userGroup);
+
+        List<String> userIds = usernames.stream()
+                .map(username -> username + "@" + domain.domainId)
+                .collect(Collectors.toList());
+
+        sharingRegistryServerHandler.addUsersToGroup(domain.domainId, userIds, userGroup.getGroupId());
+        return userGroup;
+    }
+
     private static CredentialStoreService.Client getCredentialStoreServiceClient() throws
TException, ApplicationSettingsException {
         final int serverPort = Integer.parseInt(ServerSettings.getCredentialStoreServerPort());
         final String serverHost = ServerSettings.getCredentialStoreServerHost();
diff --git a/modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/ConnectionFactory.java
b/modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/ConnectionFactory.java
index 1c41da8..f2398c4 100644
--- a/modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/ConnectionFactory.java
+++ b/modules/sharing-registry/sharing-data-migrator/src/main/java/org/apache/airavata/sharing/registry/migrator/airavata/ConnectionFactory.java
@@ -19,6 +19,8 @@
  */
 package org.apache.airavata.sharing.registry.migrator.airavata;
 
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -32,17 +34,26 @@ public class ConnectionFactory {
     //static reference to itself
     private static ConnectionFactory instance;
 
-    public static final String EXPCAT_URL = "jdbc:mysql://localhost/airavata_exp_catalog";
-    public static final String EXPCAT_USER = "root";
-    public static final String EXPCAT_PASSWORD = "";
-    public static final String DRIVER_CLASS = "com.mysql.jdbc.Driver";
+    private static final String REGISTRY_DB_URL = "registry.jdbc.url";
+    private static final String REGISTRY_DB_USER = "registry.jdbc.user";
+    private static final String REGISTRY_DB_PASSWORD = "registry.jdbc.password";
+    private static final String REGISTRY_DB_DRIVER = "registry.jdbc.driver";
+
 
     private static Connection expCatConnection;
 
     //private constructor
     private ConnectionFactory() throws ClassNotFoundException, SQLException {
-        Class.forName(DRIVER_CLASS);
-        expCatConnection = DriverManager.getConnection(EXPCAT_URL, EXPCAT_USER, EXPCAT_PASSWORD);
+        try {
+            final String EXPCAT_URL = ServerSettings.getSetting(REGISTRY_DB_URL);
+            final String EXPCAT_USER = ServerSettings.getSetting(REGISTRY_DB_USER);
+            final String EXPCAT_PASSWORD = ServerSettings.getSetting(REGISTRY_DB_PASSWORD);
+            final String DRIVER_CLASS = ServerSettings.getSetting(REGISTRY_DB_DRIVER);
+            Class.forName(DRIVER_CLASS);
+            expCatConnection = DriverManager.getConnection(EXPCAT_URL, EXPCAT_USER, EXPCAT_PASSWORD);
+        } catch (ApplicationSettingsException e) {
+            throw new RuntimeException("Failed to load application setting", e);
+        }
     }
 
     public static ConnectionFactory getInstance() throws SQLException, ClassNotFoundException
{
diff --git a/modules/sharing-registry/sharing-data-migrator/src/main/resources/.gitignore
b/modules/sharing-registry/sharing-data-migrator/src/main/resources/.gitignore
new file mode 100644
index 0000000..0e5f10f
--- /dev/null
+++ b/modules/sharing-registry/sharing-data-migrator/src/main/resources/.gitignore
@@ -0,0 +1 @@
+airavata-server.properties
diff --git a/modules/sharing-registry/sharing-data-migrator/src/main/resources/airavata-server.properties.sample
b/modules/sharing-registry/sharing-data-migrator/src/main/resources/airavata-server.properties.sample
new file mode 100644
index 0000000..cc7371b
--- /dev/null
+++ b/modules/sharing-registry/sharing-data-migrator/src/main/resources/airavata-server.properties.sample
@@ -0,0 +1,66 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+###########################################################################
+#
+#  This properties file provides configuration for all of the properties
+#  needed by the AiravataDataMigrator.
+#
+###########################################################################
+
+###########################################################################
+#  API Server Registry Configuration
+###########################################################################
+
+registry.jdbc.driver=org.mariadb.jdbc.Driver
+registry.jdbc.url=jdbc:mariadb://localhost:3306/experiment_catalog
+registry.jdbc.user=airavata
+registry.jdbc.password=airavata
+
+###########################################################################
+#  Sharing Catalog DB Configuration
+###########################################################################
+sharingcatalog.jdbc.driver=org.mariadb.jdbc.Driver
+sharingcatalog.jdbc.url=jdbc:mariadb://localhost:3306/sharing_catalog
+sharingcatalog.jdbc.user=airavata
+sharingcatalog.jdbc.password=airavata
+sharingcatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+###########################################################################
+#  Keycloak Configuration
+###########################################################################
+
+# Note: this relative path works if running the AiravataDataMigrator from the sharing-data-migrator
module directory
+trust.store=../../configuration/server/src/main/resources/client_truststore.jks
+trust.store.password=airavata
+iam.server.url=http://localhost/auth
+
+###########################################################################
+#  Registry Server Configurations
+###########################################################################
+regserver.server.host=localhost
+regserver.server.port=8970
+
+
+###########################################################################
+# Credential Store module Configuration
+###########################################################################
+credential.store.server.host=localhost
+credential.store.server.port=8960
diff --git a/modules/sharing-registry/sharing-registry-server/src/main/java/org/apache/airavata/sharing/registry/utils/ThriftDataModelConversion.java
b/modules/sharing-registry/sharing-registry-server/src/main/java/org/apache/airavata/sharing/registry/utils/ThriftDataModelConversion.java
index adef4e7..0b26a5a 100644
--- a/modules/sharing-registry/sharing-registry-server/src/main/java/org/apache/airavata/sharing/registry/utils/ThriftDataModelConversion.java
+++ b/modules/sharing-registry/sharing-registry-server/src/main/java/org/apache/airavata/sharing/registry/utils/ThriftDataModelConversion.java
@@ -19,6 +19,7 @@ public class ThriftDataModelConversion {
         User user = new User();
         user.setUserId(userProfile.getUserId());
         user.setDomainId(userProfile.getGatewayId());
+        // TODO: should set username to getUserId, but see SharingServiceDBEventHandler which
overrides userId (see AIRAVATA-2788)
         user.setUserName(userProfile.getFirstName() + " " + userProfile.getLastName());
         user.setEmail(userProfile.getEmails().get(0));
         return user;

-- 
To stop receiving notification emails like this one, please contact
machristie@apache.org.

Mime
View raw message