[NAE-2161] Optimize ElasticCaseService queries to eliminate maxClauseCount error

application-engine/src/main/java/com/netgrif/application/engine/elastic/service/ElasticCaseService.java

@@ -1,7 +1,10 @@
 package com.netgrif.application.engine.elastic.service;
 
+import co.elastic.clients.elasticsearch._types.FieldValue;
 import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField;
+import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
 import com.netgrif.application.engine.configuration.properties.DataConfigurationProperties;
 import com.netgrif.application.engine.objects.auth.domain.LoggedUser;
 import com.netgrif.application.engine.objects.elastic.domain.ElasticCase;
@@ -22,8 +25,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.actuate.autoconfigure.metrics.export.elastic.ElasticProperties;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.dao.InvalidDataAccessApiUsageException;
@@ -40,12 +41,11 @@
 import java.util.*;
 import java.util.function.BinaryOperator;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.springframework.data.elasticsearch.client.elc.Queries.matchQuery;
 import static org.springframework.data.elasticsearch.client.elc.Queries.termQuery;
 
-//import static org.elasticsearch.index.query.QueryBuilders.*;
-
 @Service
 @RequiredArgsConstructor
 public class ElasticCaseService extends ElasticViewPermissionService implements IElasticCaseService {
@@ -180,7 +180,7 @@ protected NativeQuery buildQuery(List<CaseSearchRequest> requests, LoggedUser us
             return null;
         } else if (!isIntersection) {
             singleQueries = singleQueries.stream().filter(Objects::nonNull).collect(Collectors.toList());
-            if (singleQueries.size() == 0) {
+            if (singleQueries.isEmpty()) {
                 // all queries result in an empty set => the entire result is an empty set
                 return null;
             }
@@ -214,26 +214,39 @@ protected BoolQuery.Builder buildSingleQuery(CaseSearchRequest request, LoggedUs
 
         // TODO: filtered query https://stackoverflow.com/questions/28116404/filtered-query-using-nativesearchquerybuilder-in-spring-data-elasticsearch
 
-        if (resultAlwaysEmpty)
-            return null;
-        else
-            return query;
+        return resultAlwaysEmpty ? null : query;
     }
 
     protected void buildPetriNetQuery(CaseSearchRequest request, LoggedUser user, BoolQuery.Builder query) {
         if (request.process == null || request.process.isEmpty()) {
             return;
         }
 
-        BoolQuery.Builder petriNetQuery = new BoolQuery.Builder();
+        Set<String> identifiersSet = new HashSet<>();
+        Set<String> processIdsSet = new HashSet<>();
 
-        for (CaseSearchRequest.PetriNet process : request.process) {
-            if (process.identifier != null) {
-                petriNetQuery.should(termQuery("processIdentifier", process.identifier)._toQuery());
+        request.process.forEach(p -> {
+            if (p.identifier != null) {
+                identifiersSet.add(p.identifier);
             }
-            if (process.processId != null) {
-                petriNetQuery.should(termQuery("processId", process.processId)._toQuery());
+            if (p.processId != null) {
+                processIdsSet.add(p.processId);
             }
+        });
+        TermsQueryField identifiers = new TermsQueryField.Builder()
+                .value(identifiersSet.stream().map(FieldValue::of).collect(Collectors.toList()))
+                .build();
+
+        TermsQueryField processIds = new TermsQueryField.Builder()
+                .value(processIdsSet.stream().map(FieldValue::of).collect(Collectors.toList()))
+                .build();
+
+        BoolQuery.Builder petriNetQuery = new BoolQuery.Builder();
+        if (!identifiers.value().isEmpty()) {
+            petriNetQuery.should(QueryBuilders.terms(term -> term.field("processIdentifier").terms(identifiers)));
+        }
+        if (!processIds.value().isEmpty()) {
+            petriNetQuery.should(QueryBuilders.terms(term -> term.field("processId").terms(processIds)));
         }
 
         query.filter(petriNetQuery.build()._toQuery());
@@ -307,12 +320,11 @@ protected void buildTaskQuery(CaseSearchRequest request, BoolQuery.Builder query
             return;
         }
 
-        BoolQuery.Builder taskQuery = new BoolQuery.Builder();
-        for (String taskImportId : request.transition) {
-            taskQuery.should(termQuery("taskIds", taskImportId)._toQuery());
-        }
+        TermsQueryField taskIds = new TermsQueryField.Builder()
+                .value(request.transition.stream().map(FieldValue::of).collect(Collectors.toList()))
+                .build();
 
-        query.filter(taskQuery.build()._toQuery());
+        query.filter(QueryBuilders.terms(term -> term.field("taskIds").terms(taskIds)));
     }
 
     /**
@@ -338,12 +350,11 @@ protected void buildRoleQuery(CaseSearchRequest request, BoolQuery.Builder query
             return;
         }
 
-        BoolQuery.Builder roleQuery = new BoolQuery.Builder();
-        for (String roleId : request.role) {
-            roleQuery.should(termQuery("enabledRoles", roleId)._toQuery());
-        }
+        TermsQueryField roleIds = new TermsQueryField.Builder()
+                .value(request.role.stream().map(FieldValue::of).collect(Collectors.toList()))
+                .build();
 
-        query.filter(roleQuery.build()._toQuery());
+        query.filter(QueryBuilders.terms(term -> term.field("enabledRoles").terms(roleIds)));
     }
 
     /**
@@ -434,19 +445,19 @@ protected void buildCaseIdQuery(CaseSearchRequest request, BoolQuery.Builder que
             return;
         }
 
-        BoolQuery.Builder caseIdQuery = new BoolQuery.Builder();
-        request.stringId.forEach(caseId -> caseIdQuery.should(termQuery("stringId", caseId)._toQuery()));
-        query.filter(caseIdQuery.build()._toQuery());
+        TermsQueryField stringIds = new TermsQueryField.Builder()
+                .value(request.stringId.stream().map(FieldValue::of).collect(Collectors.toList()))
+                .build();
+
+        query.filter(QueryBuilders.terms(term -> term.field("stringId").terms(stringIds)));
     }
 
     protected void buildUriNodeIdQuery(CaseSearchRequest request, BoolQuery.Builder query) {
         if (request.uriNodeId == null || request.uriNodeId.isEmpty()) {
             return;
         }
 
-        BoolQuery.Builder caseIdQuery = new BoolQuery.Builder();
-        caseIdQuery.should(termQuery("uriNodeId", request.uriNodeId)._toQuery());
-        query.filter(caseIdQuery.build()._toQuery());
+        query.filter(termQuery("uriNodeId", request.uriNodeId)._toQuery());
     }
 
     /**
@@ -475,14 +486,15 @@ protected boolean buildGroupQuery(CaseSearchRequest request, LoggedUser user, Lo
         PetriNetSearch processQuery = new PetriNetSearch();
         processQuery.setGroup(request.group);
         List<PetriNetReference> groupProcesses = this.petriNetService.search(processQuery, user, new FullPageRequest(), locale).getContent();
-        if (groupProcesses.size() == 0)
+        if (groupProcesses.isEmpty()) {
             return true;
+        }
+
+        TermsQueryField stringIds = new TermsQueryField.Builder()
+                .value(groupProcesses.stream().map(PetriNetReference::getIdentifier).map(FieldValue::of).collect(Collectors.toList()))
+                .build();
 
-        BoolQuery.Builder groupQuery = new BoolQuery.Builder();
-        groupProcesses.stream().map(PetriNetReference::getIdentifier)
-                .map(netIdentifier -> termQuery("processIdentifier", netIdentifier))
-                .forEach(termQuery -> groupQuery.should(termQuery._toQuery()));
-        query.filter(groupQuery.build()._toQuery());
+        query.filter(QueryBuilders.terms(term -> term.field("processIdentifier").terms(stringIds)));
         return false;
     }
 

application-engine/src/main/java/com/netgrif/application/engine/elastic/service/ElasticTaskService.java

@@ -1,8 +1,7 @@
 package com.netgrif.application.engine.elastic.service;
 
-import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
-import co.elastic.clients.elasticsearch._types.query_dsl.Query;
-import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery;
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.query_dsl.*;
 import com.google.common.collect.ImmutableList;
 import com.netgrif.application.engine.configuration.properties.DataConfigurationProperties;
 import com.netgrif.application.engine.objects.auth.domain.LoggedUser;
@@ -357,11 +356,13 @@ protected void buildProcessQuery(ElasticTaskSearchRequest request, BoolQuery.Bui
             return;
         }
 
+        TermsQueryField processIds = new TermsQueryField.Builder()
+                .value(request.process.stream().map(process -> process.identifier).map(FieldValue::of).collect(Collectors.toList()))
+                .build();
+
         BoolQuery.Builder processQuery = new BoolQuery.Builder();
-        for (PetriNet process : request.process) {
-            if (process.identifier != null) {
-                processQuery.should(termQuery("processId", process.identifier)._toQuery());
-            }
+        if (!processIds.value().isEmpty()) {
+            processQuery.should(QueryBuilders.terms(term -> term.field("processId").terms(processIds)));
         }
 
         query.filter(processQuery.build()._toQuery());
@@ -451,15 +452,15 @@ public boolean buildGroupQuery(TaskSearchRequest request, LoggedUser user, Local
         PetriNetSearch processQuery = new PetriNetSearch();
         processQuery.setGroup(request.group);
         List<PetriNetReference> groupProcesses = this.petriNetService.search(processQuery, user, new FullPageRequest(), locale).getContent();
-        if (groupProcesses.size() == 0)
+        if (groupProcesses.isEmpty()) {
             return true;
-
-        BoolQuery.Builder groupProcessQuery = new BoolQuery.Builder();
-        for (PetriNetReference process : groupProcesses) {
-            groupProcessQuery.should(termQuery("processId", process.getStringId())._toQuery());
         }
+        TermsQueryField stringIds = new TermsQueryField.Builder()
+                .value(groupProcesses.stream().map(PetriNetReference::getStringId).map(FieldValue::of).collect(Collectors.toList()))
+                .build();
+
+        query.filter(QueryBuilders.terms(term -> term.field("processId").terms(stringIds)));
 
-        query.filter(groupProcessQuery.build()._toQuery());
         return false;
     }
 }

application-engine/src/main/java/com/netgrif/application/engine/elastic/service/ElasticViewPermissionService.java

@@ -1,21 +1,28 @@
 package com.netgrif.application.engine.elastic.service;
 
+import co.elastic.clients.elasticsearch._types.FieldValue;
 import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
 import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
+import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField;
 import com.netgrif.application.engine.objects.auth.domain.LoggedUser;
 import com.netgrif.application.engine.objects.petrinet.domain.roles.ProcessRole;
 
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 import static org.springframework.data.elasticsearch.client.elc.Queries.termQuery;
 
 public abstract class ElasticViewPermissionService {
 
     protected void buildViewPermissionQuery(BoolQuery.Builder query, LoggedUser user) {
-        BoolQuery.Builder viewPermsExists = new BoolQuery.Builder();
-        BoolQuery.Builder viewPermNotExistsBuilder = new BoolQuery.Builder();
+        BoolQuery.Builder viewPermsExists = new BoolQuery.Builder()
+                .should(should -> should.exists(ExistsQuery.of(builder -> builder.field("viewRoles"))))
+                .should(should -> should.exists(ExistsQuery.of(builder -> builder.field("viewUserRefs"))));
+        BoolQuery.Builder viewPermNotExistsBuilder = new BoolQuery.Builder()
+                .mustNot(mustNot -> mustNot.bool(viewPermsExists.build()));
 
-        viewPermsExists.should(should -> should.exists(ExistsQuery.of(builder -> builder.field("viewRoles"))));
-        viewPermsExists.should(should -> should.exists(ExistsQuery.of(builder -> builder.field("viewUserRefs"))));
-        BoolQuery viewPermNotExists = viewPermNotExistsBuilder.mustNot(mustNot -> mustNot.bool(viewPermsExists.build())).build();
+        BoolQuery viewPermNotExists = viewPermNotExistsBuilder.build();
 
         /* Build positive view role query */
         BoolQuery positiveViewRole = buildPositiveViewRoleQuery(viewPermNotExists, user);
@@ -44,9 +51,13 @@ protected void buildViewPermissionQuery(BoolQuery.Builder query, LoggedUser user
     private BoolQuery buildPositiveViewRoleQuery(BoolQuery viewPermNotExists, LoggedUser user) {
         BoolQuery.Builder positiveViewRole = new BoolQuery.Builder();
         BoolQuery.Builder positiveViewRoleQuery = new BoolQuery.Builder();
-        for (ProcessRole role : user.getProcessRoles()) {
-            positiveViewRoleQuery.should(termQuery("viewRoles", role.getStringId())._toQuery());
-        }
+
+        TermsQueryField roleIds = new TermsQueryField.Builder()
+                .value(user.getProcessRoles().stream().map(ProcessRole::getStringId).map(FieldValue::of).collect(Collectors.toList()))
+                .build();
+
+        positiveViewRoleQuery.should(QueryBuilders.terms(term -> term.field("viewRoles").terms(roleIds)));
+
         positiveViewRole.should(viewPermNotExists._toQuery());
         positiveViewRole.should(positiveViewRoleQuery.build()._toQuery());
         return positiveViewRole.build();
@@ -55,41 +66,41 @@ private BoolQuery buildPositiveViewRoleQuery(BoolQuery viewPermNotExists, Logged
     private BoolQuery buildNegativeViewRoleQuery(LoggedUser user) {
         BoolQuery.Builder negativeViewRole = new BoolQuery.Builder();
         BoolQuery.Builder negativeViewRoleQuery = new BoolQuery.Builder();
-        for (ProcessRole role : user.getProcessRoles()) {
-            negativeViewRoleQuery.should(termQuery("negativeViewRoles", role.getStringId())._toQuery());
-        }
+
+        TermsQueryField roleIds = new TermsQueryField.Builder()
+                .value(user.getProcessRoles().stream().map(ProcessRole::getStringId).map(FieldValue::of).collect(Collectors.toList()))
+                .build();
+
+        negativeViewRoleQuery.should(QueryBuilders.terms(term -> term.field("negativeViewRoles").terms(roleIds)));
         negativeViewRole.mustNot(negativeViewRoleQuery.build()._toQuery());
         return negativeViewRole.build();
     }
 
     private BoolQuery buildPositiveViewUser(BoolQuery viewPermNotExists, LoggedUser user) {
-        BoolQuery.Builder positiveViewUser = new BoolQuery.Builder();
-        BoolQuery.Builder positiveViewUserQuery = new BoolQuery.Builder();
-        positiveViewUserQuery.must(termQuery("viewUsers", user.getStringId())._toQuery());
-        positiveViewUser.should(viewPermNotExists._toQuery());
-        positiveViewUser.should(positiveViewUserQuery.build()._toQuery());
-        return positiveViewUser.build();
+        return new BoolQuery.Builder()
+                .should(viewPermNotExists._toQuery())
+                .filter(termQuery("viewUsers", user.getStringId())._toQuery())
+                .build();
     }
 
     private BoolQuery buildNegativeViewUser(LoggedUser user) {
-        BoolQuery.Builder negativeViewUser = new BoolQuery.Builder();
-        BoolQuery.Builder negativeViewUserQuery = new BoolQuery.Builder();
-        negativeViewUserQuery.should(termQuery("negativeViewUsers", user.getStringId())._toQuery());
-        negativeViewUser.mustNot(negativeViewUserQuery.build()._toQuery());
-        return negativeViewUser.build();
+        return new BoolQuery.Builder()
+                .mustNot(termQuery("negativeViewUsers", user.getStringId())._toQuery())
+                .build();
     }
 
     private BoolQuery setMinus(BoolQuery positiveSet, BoolQuery negativeSet) {
-        BoolQuery.Builder positiveSetMinusNegativeSet = new BoolQuery.Builder();
-        positiveSetMinusNegativeSet.must(positiveSet._toQuery());
-        positiveSetMinusNegativeSet.must(negativeSet._toQuery());
-        return positiveSetMinusNegativeSet.build();
+        return new BoolQuery.Builder()
+                .must(positiveSet._toQuery())
+                .must(negativeSet._toQuery())
+                .build();
     }
 
     private BoolQuery union(BoolQuery setA, BoolQuery setB) {
-        BoolQuery.Builder unionSet = new BoolQuery.Builder();
-        unionSet.should(setA._toQuery());
-        unionSet.should(setB._toQuery());
-        return unionSet.build();
+        return new BoolQuery.Builder()
+                .should(setA._toQuery())
+                .should(setB._toQuery())
+                .minimumShouldMatch(String.valueOf(1))
+                .build();
     }
 }

application-engine/src/main/java/com/netgrif/application/engine/workflow/service/CaseSearchService.java

@@ -142,10 +142,8 @@ public Predicate negativeViewUserQuery(String userId) {
     public Predicate petriNet(Object query, LoggedUser user, Locale locale) {
         List<PetriNetReference> allowedNets = petriNetService.getReferencesByUsersProcessRoles(user, locale);
         if (query instanceof ArrayList) {
-            BooleanBuilder builder = new BooleanBuilder();
-            List<BooleanExpression> expressions = (List<BooleanExpression>) ((ArrayList) query).stream().filter(q -> q instanceof HashMap).map(q -> petriNetObject((HashMap<String, String>) q, allowedNets)).collect(Collectors.toList());
-            expressions.forEach(builder::or);
-            return builder;
+            List<Predicate> expressions = (List<Predicate>) ((ArrayList) query).stream().filter(q -> q instanceof HashMap).map(q -> petriNetObject((HashMap<String, String>) q, allowedNets)).collect(Collectors.toList());
+            return constructPredicateTree(expressions, BooleanBuilder::or);
         } else if (query instanceof HashMap) {
             return petriNetObject((HashMap<String, String>) query, allowedNets);
         }
@@ -346,10 +344,8 @@ private String parseProcessIdentifier(HashMap<String, String> petriNet) {
 
     public Predicate caseId(Object query) {
         if (query instanceof ArrayList) {
-            BooleanBuilder builder = new BooleanBuilder();
-            List<BooleanExpression> expressions = (List<BooleanExpression>) ((ArrayList) query).stream().filter(q -> q instanceof String).map(q -> caseIdString((String) q)).collect(Collectors.toList());
-            expressions.forEach(builder::or);
-            return builder;
+            List<Predicate> expressions = (List<Predicate>) ((ArrayList) query).stream().filter(q -> q instanceof String).map(q -> caseIdString((String) q)).collect(Collectors.toList());
+            return constructPredicateTree(expressions, BooleanBuilder::or);
         } else if (query instanceof String) {
             return caseIdString((String) query);
         }
@@ -371,9 +367,7 @@ public Predicate group(Object query, LoggedUser user, Locale locale) {
         if (groupProcesses.size() == 0)
             return null;
 
-        List<BooleanExpression> processQueries = groupProcesses.stream().map(PetriNetReference::getIdentifier).map(QCase.case$.processIdentifier::eq).collect(Collectors.toList());
-        BooleanBuilder builder = new BooleanBuilder();
-        processQueries.forEach(builder::or);
-        return builder;
+        List<Predicate> processQueries = groupProcesses.stream().map(PetriNetReference::getIdentifier).map(QCase.case$.processIdentifier::eq).collect(Collectors.toList());
+        return constructPredicateTree(processQueries, BooleanBuilder::or);
     }
 }