syncope-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ilgro...@apache.org
Subject [syncope] branch master updated: Resolving concurrent update / delete issues under high load
Date Mon, 19 Nov 2018 15:17:54 GMT
This is an automated email from the ASF dual-hosted git repository.

ilgrosso pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/syncope.git


The following commit(s) were added to refs/heads/master by this push:
     new 4eceb6d  Resolving concurrent update / delete issues under high load
4eceb6d is described below

commit 4eceb6d775de9b4b3804b4622455987f63fd6621
Author: Francesco Chicchiriccò <ilgrosso@apache.org>
AuthorDate: Mon Nov 19 16:17:35 2018 +0100

    Resolving concurrent update / delete issues under high load
---
 .../core/persistence/jpa/dao/JPAAnyObjectDAO.java  |   7 +-
 .../core/persistence/jpa/dao/JPAGroupDAO.java      | 143 +++++++++++++--------
 .../core/persistence/jpa/dao/JPARoleDAO.java       |  51 +++++---
 .../core/persistence/jpa/dao/JPAUserDAO.java       |  14 +-
 4 files changed, 130 insertions(+), 85 deletions(-)

diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAAnyObjectDAO.java
b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAAnyObjectDAO.java
index f4aacd2..8035812 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAAnyObjectDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAAnyObjectDAO.java
@@ -268,11 +268,10 @@ public class JPAAnyObjectDAO extends AbstractAnyDAO<AnyObject>
implements AnyObj
         query.getResultList().stream().map(resultKey -> resultKey instanceof Object[]
                 ? (String) ((Object[]) resultKey)[0]
                 : ((String) resultKey)).
-                forEachOrdered(actualKey -> {
-                    Group group = groupDAO.find(actualKey.toString());
+                forEach(groupKey -> {
+                    Group group = groupDAO.find(groupKey.toString());
                     if (group == null) {
-                        LOG.error("Could not find group with id {}, even though returned
by the native query",
-                                actualKey);
+                        LOG.error("Could not find group {}, even though returned by the native
query", groupKey);
                     } else if (!result.contains(group)) {
                         result.add(group);
                     }
diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAGroupDAO.java
b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAGroupDAO.java
index a343ebc..5700453 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAGroupDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAGroupDAO.java
@@ -288,7 +288,7 @@ public class JPAGroupDAO extends AbstractAnyDAO<Group> implements
GroupDAO {
             }
         }
         clearADynMembers(merged);
-        merged.getADynMemberships().stream().forEach(memb -> {
+        merged.getADynMemberships().forEach(memb -> {
             SearchCond cond = buildDynMembershipCond(memb.getFIQLCond(), merged.getRealm());
             int count = searchDAO.count(
                     Collections.<String>singleton(merged.getRealm().getFullPath()),
cond, AnyTypeKind.ANY_OBJECT);
@@ -327,12 +327,12 @@ public class JPAGroupDAO extends AbstractAnyDAO<Group> implements
GroupDAO {
             AnyObject leftEnd = membership.getLeftEnd();
             leftEnd.remove(membership);
             membership.setRightEnd(null);
-            leftEnd.getPlainAttrs(membership).stream().map(attr -> {
+            leftEnd.getPlainAttrs(membership).forEach(attr -> {
                 leftEnd.remove(attr);
                 attr.setOwner(null);
                 attr.setMembership(null);
-                return attr;
-            }).forEachOrdered(attr -> plainAttrDAO.delete(attr));
+                plainAttrDAO.delete(attr);
+            });
 
             anyObjectDAO.save(leftEnd);
             publisher.publishEvent(new AnyCreatedUpdatedEvent<>(this, leftEnd, AuthContextUtils.getDomain()));
@@ -342,12 +342,13 @@ public class JPAGroupDAO extends AbstractAnyDAO<Group> implements
GroupDAO {
             User leftEnd = membership.getLeftEnd();
             leftEnd.remove(membership);
             membership.setRightEnd(null);
-            leftEnd.getPlainAttrs(membership).stream().map(attr -> {
+            leftEnd.getPlainAttrs(membership).forEach(attr -> {
                 leftEnd.remove(attr);
                 attr.setOwner(null);
                 attr.setMembership(null);
-                return attr;
-            }).forEachOrdered(attr -> plainAttrDAO.delete(attr));
+
+                plainAttrDAO.delete(attr);
+            });
 
             userDAO.save(leftEnd);
             publisher.publishEvent(new AnyCreatedUpdatedEvent<>(this, leftEnd, AuthContextUtils.getDomain()));
@@ -375,17 +376,20 @@ public class JPAGroupDAO extends AbstractAnyDAO<Group> implements
GroupDAO {
     @SuppressWarnings("unchecked")
     public List<String> findADynMembers(final Group group) {
         List<String> result = new ArrayList<>();
-        group.getADynMemberships().stream().map(memb -> {
+
+        group.getADynMemberships().forEach(memb -> {
             Query query = entityManager().createNativeQuery(
                     "SELECT any_id FROM " + ADYNMEMB_TABLE + " WHERE group_id=? AND anyType_id=?");
             query.setParameter(1, group.getKey());
             query.setParameter(2, memb.getAnyType().getKey());
-            return query;
-        }).forEachOrdered((query) -> {
+
             query.getResultList().stream().map(key -> key instanceof Object[]
                     ? (String) ((Object[]) key)[0]
                     : ((String) key)).
-                    forEachOrdered(actualKey -> result.add(actualKey.toString()));
+                    filter(anyObject -> !result.contains((String) anyObject)).
+                    forEach(anyObject -> {
+                        result.add((String) anyObject);
+                    });
         });
 
         return result;
@@ -448,35 +452,47 @@ public class JPAGroupDAO extends AbstractAnyDAO<Group> implements
GroupDAO {
 
     @Transactional
     @Override
+    @SuppressWarnings("unchecked")
     public Pair<Set<String>, Set<String>> refreshDynMemberships(final AnyObject
anyObject) {
+        Query query = entityManager().createNativeQuery(
+                "SELECT group_id FROM " + JPAGroupDAO.ADYNMEMB_TABLE + " WHERE any_id=?");
+        query.setParameter(1, anyObject.getKey());
+
         Set<String> before = new HashSet<>();
+        query.getResultList().stream().
+                map(resultKey -> resultKey instanceof Object[]
+                ? (String) ((Object[]) resultKey)[0]
+                : ((String) resultKey)).
+                forEach(group -> before.add((String) group));
+
         Set<String> after = new HashSet<>();
-        findWithADynMemberships(anyObject.getType()).stream().map(memb -> {
+        findWithADynMemberships(anyObject.getType()).stream().
+                filter(membCond -> jpaAnySearchDAO().matches(
+                anyObject,
+                buildDynMembershipCond(membCond.getFIQLCond(), membCond.getGroup().getRealm()))).
+                forEach(membCond -> {
+                    if (!before.contains(membCond.getGroup().getKey())) {
+                        Query insert = entityManager().createNativeQuery(
+                                "INSERT INTO " + ADYNMEMB_TABLE + " VALUES(?, ?, ?)");
+                        insert.setParameter(1, anyObject.getType().getKey());
+                        insert.setParameter(2, anyObject.getKey());
+                        insert.setParameter(3, membCond.getGroup().getKey());
+                        insert.executeUpdate();
+                    }
+
+                    after.add(membCond.getGroup().getKey());
+
+                    publisher.publishEvent(
+                            new AnyCreatedUpdatedEvent<>(this, membCond.getGroup(),
AuthContextUtils.getDomain()));
+                });
+
+        before.stream().filter(group -> !after.contains(group)).forEach(group -> {
             Query delete = entityManager().createNativeQuery(
                     "DELETE FROM " + ADYNMEMB_TABLE + " WHERE group_id=? AND any_id=?");
-            delete.setParameter(1, memb.getGroup().getKey());
+            delete.setParameter(1, group);
             delete.setParameter(2, anyObject.getKey());
-
-            if (delete.executeUpdate() > 0) {
-                before.add(memb.getGroup().getKey());
-            }
-
-            if (jpaAnySearchDAO().matches(
-                    anyObject,
-                    buildDynMembershipCond(memb.getFIQLCond(), memb.getGroup().getRealm())))
{
-
-                Query insert = entityManager().createNativeQuery(
-                        "INSERT INTO " + ADYNMEMB_TABLE + " VALUES(?, ?, ?)");
-                insert.setParameter(1, anyObject.getType().getKey());
-                insert.setParameter(2, anyObject.getKey());
-                insert.setParameter(3, memb.getGroup().getKey());
-                insert.executeUpdate();
-
-                after.add(memb.getGroup().getKey());
-            }
-            return memb;
-        }).forEachOrdered(memb -> publisher.publishEvent(
-                new AnyCreatedUpdatedEvent<>(this, memb.getGroup(), AuthContextUtils.getDomain())));
+            delete.executeUpdate();
+        });
 
         return Pair.of(before, after);
     }
@@ -513,7 +529,8 @@ public class JPAGroupDAO extends AbstractAnyDAO<Group> implements
GroupDAO {
         List<String> result = new ArrayList<>();
         query.getResultList().stream().map(key -> key instanceof Object[]
                 ? (String) ((Object[]) key)[0]
-                : ((String) key)).forEachOrdered(actualKey -> result.add(actualKey.toString()));
+                : ((String) key)).
+                forEach(user -> result.add((String) user));
         return result;
     }
 
@@ -534,34 +551,46 @@ public class JPAGroupDAO extends AbstractAnyDAO<Group> implements
GroupDAO {
 
     @Transactional
     @Override
+    @SuppressWarnings("unchecked")
     public Pair<Set<String>, Set<String>> refreshDynMemberships(final User
user) {
+        Query query = entityManager().createNativeQuery(
+                "SELECT group_id FROM " + JPAGroupDAO.UDYNMEMB_TABLE + " WHERE any_id=?");
+        query.setParameter(1, user.getKey());
+
         Set<String> before = new HashSet<>();
+        query.getResultList().stream().
+                map(resultKey -> resultKey instanceof Object[]
+                ? (String) ((Object[]) resultKey)[0]
+                : ((String) resultKey)).
+                forEach(group -> before.add((String) group));
+
         Set<String> after = new HashSet<>();
-        findWithUDynMemberships().stream().map(memb -> {
+        findWithUDynMemberships().stream().
+                filter(membCond -> jpaAnySearchDAO().matches(
+                user,
+                buildDynMembershipCond(membCond.getFIQLCond(), membCond.getGroup().getRealm()))).
+                forEach(membCond -> {
+                    if (!before.contains(membCond.getGroup().getKey())) {
+                        Query insert = entityManager().createNativeQuery(
+                                "INSERT INTO " + UDYNMEMB_TABLE + " VALUES(?, ?)");
+                        insert.setParameter(1, user.getKey());
+                        insert.setParameter(2, membCond.getGroup().getKey());
+                        insert.executeUpdate();
+                    }
+
+                    after.add(membCond.getGroup().getKey());
+
+                    publisher.publishEvent(
+                            new AnyCreatedUpdatedEvent<>(this, membCond.getGroup(),
AuthContextUtils.getDomain()));
+                });
+
+        before.stream().filter(group -> !after.contains(group)).forEach(group -> {
             Query delete = entityManager().createNativeQuery(
                     "DELETE FROM " + UDYNMEMB_TABLE + " WHERE group_id=? AND any_id=?");
-            delete.setParameter(1, memb.getGroup().getKey());
+            delete.setParameter(1, group);
             delete.setParameter(2, user.getKey());
-
-            if (delete.executeUpdate() > 0) {
-                before.add(memb.getGroup().getKey());
-            }
-
-            if (jpaAnySearchDAO().matches(
-                    user,
-                    buildDynMembershipCond(memb.getFIQLCond(), memb.getGroup().getRealm())))
{
-
-                Query insert = entityManager().createNativeQuery(
-                        "INSERT INTO " + UDYNMEMB_TABLE + " VALUES(?, ?)");
-                insert.setParameter(1, user.getKey());
-                insert.setParameter(2, memb.getGroup().getKey());
-                insert.executeUpdate();
-
-                after.add(memb.getGroup().getKey());
-            }
-            return memb;
-        }).forEachOrdered(memb -> publisher.publishEvent(
-                new AnyCreatedUpdatedEvent<>(this, memb.getGroup(), AuthContextUtils.getDomain())));
+            delete.executeUpdate();
+        });
 
         return Pair.of(before, after);
     }
diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPARoleDAO.java
b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPARoleDAO.java
index 4e8295d..1cbf9ba 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPARoleDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPARoleDAO.java
@@ -20,7 +20,9 @@ package org.apache.syncope.core.persistence.jpa.dao;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import javax.persistence.Query;
 import javax.persistence.TypedQuery;
 import org.apache.syncope.common.lib.types.AnyTypeKind;
@@ -142,6 +144,7 @@ public class JPARoleDAO extends AbstractDAO<Role> implements RoleDAO
{
     }
 
     @Override
+    @SuppressWarnings("unchecked")
     public List<String> findDynMembers(final Role role) {
         if (role.getDynMembership() == null) {
             return Collections.emptyList();
@@ -151,13 +154,10 @@ public class JPARoleDAO extends AbstractDAO<Role> implements RoleDAO
{
         query.setParameter(1, role.getKey());
 
         List<String> result = new ArrayList<>();
-        for (Object key : query.getResultList()) {
-            String actualKey = key instanceof Object[]
-                    ? (String) ((Object[]) key)[0]
-                    : ((String) key);
-
-            result.add(actualKey);
-        }
+        query.getResultList().stream().map(key -> key instanceof Object[]
+                ? (String) ((Object[]) key)[0]
+                : ((String) key)).
+                forEach(user -> result.add((String) user));
         return result;
     }
 
@@ -170,20 +170,39 @@ public class JPARoleDAO extends AbstractDAO<Role> implements RoleDAO
{
 
     @Transactional
     @Override
+    @SuppressWarnings("unchecked")
     public void refreshDynMemberships(final User user) {
-        findAll().stream().filter(role -> role.getDynMembership() != null).forEach(role
-> {
+        Query query = entityManager().createNativeQuery(
+                "SELECT role_id FROM " + DYNMEMB_TABLE + " WHERE any_id=?");
+        query.setParameter(1, user.getKey());
+
+        Set<String> before = new HashSet<>();
+        query.getResultList().stream().
+                map(resultKey -> resultKey instanceof Object[]
+                ? (String) ((Object[]) resultKey)[0]
+                : ((String) resultKey)).
+                forEach(role -> before.add((String) role));
+
+        Set<String> after = new HashSet<>();
+        findAll().stream().
+                filter(role -> role.getDynMembership() != null
+                && searchDAO.matches(user, SearchCondConverter.convert(role.getDynMembership().getFIQLCond()))
+                && !before.contains(role.getKey())).
+                forEach(role -> {
+                    Query insert = entityManager().createNativeQuery("INSERT INTO " + DYNMEMB_TABLE
+ " VALUES(?, ?)");
+                    insert.setParameter(1, user.getKey());
+                    insert.setParameter(2, role.getKey());
+                    insert.executeUpdate();
+
+                    after.add(role.getKey());
+                });
+
+        before.stream().filter(role -> !after.contains(role)).forEach(role -> {
             Query delete = entityManager().createNativeQuery(
                     "DELETE FROM " + DYNMEMB_TABLE + " WHERE role_id=? AND any_id=?");
-            delete.setParameter(1, role.getKey());
+            delete.setParameter(1, role);
             delete.setParameter(2, user.getKey());
             delete.executeUpdate();
-
-            if (searchDAO.matches(user, SearchCondConverter.convert(role.getDynMembership().getFIQLCond())))
{
-                Query insert = entityManager().createNativeQuery("INSERT INTO " + DYNMEMB_TABLE
+ " VALUES(?, ?)");
-                insert.setParameter(1, user.getKey());
-                insert.setParameter(2, role.getKey());
-                insert.executeUpdate();
-            }
         });
     }
 
diff --git a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAUserDAO.java
b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAUserDAO.java
index e86905f..01e90c7 100644
--- a/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAUserDAO.java
+++ b/core/persistence-jpa/src/main/java/org/apache/syncope/core/persistence/jpa/dao/JPAUserDAO.java
@@ -448,11 +448,10 @@ public class JPAUserDAO extends AbstractAnyDAO<User> implements
UserDAO {
         query.getResultList().stream().map(resultKey -> resultKey instanceof Object[]
                 ? (String) ((Object[]) resultKey)[0]
                 : ((String) resultKey)).
-                forEachOrdered(actualKey -> {
-                    Role role = roleDAO.find(actualKey.toString());
+                forEachOrdered(roleKey -> {
+                    Role role = roleDAO.find(roleKey.toString());
                     if (role == null) {
-                        LOG.error("Could not find role with id {}, even though returned by
the native query",
-                                actualKey);
+                        LOG.error("Could not find role {}, even though returned by the native
query", roleKey);
                     } else if (!result.contains(role)) {
                         result.add(role);
                     }
@@ -472,11 +471,10 @@ public class JPAUserDAO extends AbstractAnyDAO<User> implements
UserDAO {
         query.getResultList().stream().map(resultKey -> resultKey instanceof Object[]
                 ? (String) ((Object[]) resultKey)[0]
                 : ((String) resultKey)).
-                forEachOrdered(actualKey -> {
-                    Group group = groupDAO.find(actualKey.toString());
+                forEach(groupKey -> {
+                    Group group = groupDAO.find(groupKey.toString());
                     if (group == null) {
-                        LOG.error("Could not find group with id {}, even though returned
by the native query",
-                                actualKey);
+                        LOG.error("Could not find group {}, even though returned by the native
query", groupKey);
                     } else if (!result.contains(group)) {
                         result.add(group);
                     }


Mime
View raw message