Skip to content

[NAE-2161] Optimize ElasticCaseService queries to eliminate maxClauseCount error #335

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 20, 2025
Merged

[NAE-2161] Optimize ElasticCaseService queries to eliminate maxClauseCount error #335

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
611e3dd
Bump version to 7.0.0-RC5 across all modules
machacjozef Aug 5, 2025
5e16c99
[NAE-2161] Optimize ElasticCaseService queries to eliminate maxClause…
Kovy95 Aug 7, 2025
bbb920e
Merge remote-tracking branch 'origin/release/7.0.0-rc5' into NAE-2161
Kovy95 Aug 7, 2025
faa1569
Merge remote-tracking branch 'origin/release/7.0.0-rev6' into NAE-2161
machacjozef Aug 20, 2025
670dd19
[NAE-2161]
machacjozef Aug 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion application-engine/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<parent>
<groupId>com.netgrif</groupId>
<artifactId>application-engine-parent</artifactId>
<version>7.0.0-SNAPSHOT</version>
<version>7.0.0-RC5</version>
</parent>

<artifactId>application-engine</artifactId>
Expand Down
86 changes: 49 additions & 37 deletions ...gine/src/main/java/com/netgrif/application/engine/elastic/service/ElasticCaseService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package com.netgrif.application.engine.elastic.service;

import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import com.netgrif.application.engine.configuration.properties.DataConfigurationProperties;
import com.netgrif.application.engine.objects.auth.domain.LoggedUser;
import com.netgrif.application.engine.objects.elastic.domain.ElasticCase;
Expand All @@ -22,8 +25,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.actuate.autoconfigure.metrics.export.elastic.ElasticProperties;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.Lazy;
import org.springframework.dao.InvalidDataAccessApiUsageException;
Expand All @@ -40,12 +41,11 @@
import java.util.*;
import java.util.function.BinaryOperator;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.springframework.data.elasticsearch.client.elc.Queries.matchQuery;
import static org.springframework.data.elasticsearch.client.elc.Queries.termQuery;

//import static org.elasticsearch.index.query.QueryBuilders.*;

@Service
@RequiredArgsConstructor
public class ElasticCaseService extends ElasticViewPermissionService implements IElasticCaseService {
Expand Down Expand Up @@ -180,7 +180,7 @@ protected NativeQuery buildQuery(List<CaseSearchRequest> requests, LoggedUser us
return null;
} else if (!isIntersection) {
singleQueries = singleQueries.stream().filter(Objects::nonNull).collect(Collectors.toList());
if (singleQueries.size() == 0) {
if (singleQueries.isEmpty()) {
// all queries result in an empty set => the entire result is an empty set
return null;
}
Expand Down Expand Up @@ -214,26 +214,39 @@ protected BoolQuery.Builder buildSingleQuery(CaseSearchRequest request, LoggedUs

// TODO: filtered query https://stackoverflow.com/questions/28116404/filtered-query-using-nativesearchquerybuilder-in-spring-data-elasticsearch

if (resultAlwaysEmpty)
return null;
else
return query;
return resultAlwaysEmpty ? null : query;
}

protected void buildPetriNetQuery(CaseSearchRequest request, LoggedUser user, BoolQuery.Builder query) {
if (request.process == null || request.process.isEmpty()) {
return;
}

BoolQuery.Builder petriNetQuery = new BoolQuery.Builder();
Set<String> identifiersSet = new HashSet<>();
Set<String> processIdsSet = new HashSet<>();

for (CaseSearchRequest.PetriNet process : request.process) {
if (process.identifier != null) {
petriNetQuery.should(termQuery("processIdentifier", process.identifier)._toQuery());
request.process.forEach(p -> {
if (p.identifier != null) {
identifiersSet.add(p.identifier);
}
if (process.processId != null) {
petriNetQuery.should(termQuery("processId", process.processId)._toQuery());
if (p.processId != null) {
processIdsSet.add(p.processId);
}
});
TermsQueryField identifiers = new TermsQueryField.Builder()
.value(identifiersSet.stream().map(FieldValue::of).collect(Collectors.toList()))
.build();

TermsQueryField processIds = new TermsQueryField.Builder()
.value(processIdsSet.stream().map(FieldValue::of).collect(Collectors.toList()))
.build();

BoolQuery.Builder petriNetQuery = new BoolQuery.Builder();
if (!identifiers.value().isEmpty()) {
petriNetQuery.should(QueryBuilders.terms(term -> term.field("processIdentifier").terms(identifiers)));
}
if (!processIds.value().isEmpty()) {
petriNetQuery.should(QueryBuilders.terms(term -> term.field("processId").terms(processIds)));
}

query.filter(petriNetQuery.build()._toQuery());
Expand Down Expand Up @@ -307,12 +320,11 @@ protected void buildTaskQuery(CaseSearchRequest request, BoolQuery.Builder query
return;
}

BoolQuery.Builder taskQuery = new BoolQuery.Builder();
for (String taskImportId : request.transition) {
taskQuery.should(termQuery("taskIds", taskImportId)._toQuery());
}
TermsQueryField taskIds = new TermsQueryField.Builder()
.value(request.transition.stream().map(FieldValue::of).collect(Collectors.toList()))
.build();

query.filter(taskQuery.build()._toQuery());
query.filter(QueryBuilders.terms(term -> term.field("taskIds").terms(taskIds)));
}

/**
Expand All @@ -338,12 +350,11 @@ protected void buildRoleQuery(CaseSearchRequest request, BoolQuery.Builder query
return;
}

BoolQuery.Builder roleQuery = new BoolQuery.Builder();
for (String roleId : request.role) {
roleQuery.should(termQuery("enabledRoles", roleId)._toQuery());
}
TermsQueryField roleIds = new TermsQueryField.Builder()
.value(request.role.stream().map(FieldValue::of).collect(Collectors.toList()))
.build();

query.filter(roleQuery.build()._toQuery());
query.filter(QueryBuilders.terms(term -> term.field("enabledRoles").terms(roleIds)));
}

/**
Expand Down Expand Up @@ -434,19 +445,19 @@ protected void buildCaseIdQuery(CaseSearchRequest request, BoolQuery.Builder que
return;
}

BoolQuery.Builder caseIdQuery = new BoolQuery.Builder();
request.stringId.forEach(caseId -> caseIdQuery.should(termQuery("stringId", caseId)._toQuery()));
query.filter(caseIdQuery.build()._toQuery());
TermsQueryField stringIds = new TermsQueryField.Builder()
.value(request.stringId.stream().map(FieldValue::of).collect(Collectors.toList()))
.build();

query.filter(QueryBuilders.terms(term -> term.field("stringId").terms(stringIds)));
}

protected void buildUriNodeIdQuery(CaseSearchRequest request, BoolQuery.Builder query) {
if (request.uriNodeId == null || request.uriNodeId.isEmpty()) {
return;
}

BoolQuery.Builder caseIdQuery = new BoolQuery.Builder();
caseIdQuery.should(termQuery("uriNodeId", request.uriNodeId)._toQuery());
query.filter(caseIdQuery.build()._toQuery());
query.filter(termQuery("uriNodeId", request.uriNodeId)._toQuery());
}

/**
Expand Down Expand Up @@ -475,14 +486,15 @@ protected boolean buildGroupQuery(CaseSearchRequest request, LoggedUser user, Lo
PetriNetSearch processQuery = new PetriNetSearch();
processQuery.setGroup(request.group);
List<PetriNetReference> groupProcesses = this.petriNetService.search(processQuery, user, new FullPageRequest(), locale).getContent();
if (groupProcesses.size() == 0)
if (groupProcesses.isEmpty()) {
return true;
}

TermsQueryField stringIds = new TermsQueryField.Builder()
.value(groupProcesses.stream().map(PetriNetReference::getIdentifier).map(FieldValue::of).collect(Collectors.toList()))
.build();

BoolQuery.Builder groupQuery = new BoolQuery.Builder();
groupProcesses.stream().map(PetriNetReference::getIdentifier)
.map(netIdentifier -> termQuery("processIdentifier", netIdentifier))
.forEach(termQuery -> groupQuery.should(termQuery._toQuery()));
query.filter(groupQuery.build()._toQuery());
query.filter(QueryBuilders.terms(term -> term.field("processIdentifier").terms(stringIds)));
return false;
}

Expand Down
27 changes: 14 additions & 13 deletions ...gine/src/main/java/com/netgrif/application/engine/elastic/service/ElasticTaskService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package com.netgrif.application.engine.elastic.service;

import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryStringQuery;
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.query_dsl.*;
import com.google.common.collect.ImmutableList;
import com.netgrif.application.engine.configuration.properties.DataConfigurationProperties;
import com.netgrif.application.engine.objects.auth.domain.LoggedUser;
Expand Down Expand Up @@ -357,11 +356,13 @@ protected void buildProcessQuery(ElasticTaskSearchRequest request, BoolQuery.Bui
return;
}

TermsQueryField processIds = new TermsQueryField.Builder()
.value(request.process.stream().map(process -> process.identifier).map(FieldValue::of).collect(Collectors.toList()))
.build();

BoolQuery.Builder processQuery = new BoolQuery.Builder();
for (PetriNet process : request.process) {
if (process.identifier != null) {
processQuery.should(termQuery("processId", process.identifier)._toQuery());
}
if (!processIds.value().isEmpty()) {
processQuery.should(QueryBuilders.terms(term -> term.field("processId").terms(processIds)));
}

query.filter(processQuery.build()._toQuery());
Expand Down Expand Up @@ -451,15 +452,15 @@ public boolean buildGroupQuery(TaskSearchRequest request, LoggedUser user, Local
PetriNetSearch processQuery = new PetriNetSearch();
processQuery.setGroup(request.group);
List<PetriNetReference> groupProcesses = this.petriNetService.search(processQuery, user, new FullPageRequest(), locale).getContent();
if (groupProcesses.size() == 0)
if (groupProcesses.isEmpty()) {
return true;

BoolQuery.Builder groupProcessQuery = new BoolQuery.Builder();
for (PetriNetReference process : groupProcesses) {
groupProcessQuery.should(termQuery("processId", process.getStringId())._toQuery());
}
TermsQueryField stringIds = new TermsQueryField.Builder()
.value(groupProcesses.stream().map(PetriNetReference::getStringId).map(FieldValue::of).collect(Collectors.toList()))
.build();

query.filter(QueryBuilders.terms(term -> term.field("processId").terms(stringIds)));

query.filter(groupProcessQuery.build()._toQuery());
return false;
}
}
71 changes: 41 additions & 30 deletions ...ain/java/com/netgrif/application/engine/elastic/service/ElasticViewPermissionService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,28 @@
package com.netgrif.application.engine.elastic.service;

import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.ExistsQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField;
import com.netgrif.application.engine.objects.auth.domain.LoggedUser;
import com.netgrif.application.engine.objects.petrinet.domain.roles.ProcessRole;

import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.springframework.data.elasticsearch.client.elc.Queries.termQuery;

public abstract class ElasticViewPermissionService {

protected void buildViewPermissionQuery(BoolQuery.Builder query, LoggedUser user) {
BoolQuery.Builder viewPermsExists = new BoolQuery.Builder();
BoolQuery.Builder viewPermNotExistsBuilder = new BoolQuery.Builder();
BoolQuery.Builder viewPermsExists = new BoolQuery.Builder()
.should(should -> should.exists(ExistsQuery.of(builder -> builder.field("viewRoles"))))
.should(should -> should.exists(ExistsQuery.of(builder -> builder.field("viewUserRefs"))));
BoolQuery.Builder viewPermNotExistsBuilder = new BoolQuery.Builder()
.mustNot(mustNot -> mustNot.bool(viewPermsExists.build()));

viewPermsExists.should(should -> should.exists(ExistsQuery.of(builder -> builder.field("viewRoles"))));
viewPermsExists.should(should -> should.exists(ExistsQuery.of(builder -> builder.field("viewUserRefs"))));
BoolQuery viewPermNotExists = viewPermNotExistsBuilder.mustNot(mustNot -> mustNot.bool(viewPermsExists.build())).build();
BoolQuery viewPermNotExists = viewPermNotExistsBuilder.build();

/* Build positive view role query */
BoolQuery positiveViewRole = buildPositiveViewRoleQuery(viewPermNotExists, user);
Expand Down Expand Up @@ -44,9 +51,13 @@ protected void buildViewPermissionQuery(BoolQuery.Builder query, LoggedUser user
private BoolQuery buildPositiveViewRoleQuery(BoolQuery viewPermNotExists, LoggedUser user) {
BoolQuery.Builder positiveViewRole = new BoolQuery.Builder();
BoolQuery.Builder positiveViewRoleQuery = new BoolQuery.Builder();
for (ProcessRole role : user.getProcessRoles()) {
positiveViewRoleQuery.should(termQuery("viewRoles", role.getStringId())._toQuery());
}

TermsQueryField roleIds = new TermsQueryField.Builder()
.value(user.getProcessRoles().stream().map(ProcessRole::getStringId).map(FieldValue::of).collect(Collectors.toList()))
.build();

positiveViewRoleQuery.should(QueryBuilders.terms(term -> term.field("viewRoles").terms(roleIds)));

positiveViewRole.should(viewPermNotExists._toQuery());
positiveViewRole.should(positiveViewRoleQuery.build()._toQuery());
return positiveViewRole.build();
Expand All @@ -55,41 +66,41 @@ private BoolQuery buildPositiveViewRoleQuery(BoolQuery viewPermNotExists, Logged
private BoolQuery buildNegativeViewRoleQuery(LoggedUser user) {
BoolQuery.Builder negativeViewRole = new BoolQuery.Builder();
BoolQuery.Builder negativeViewRoleQuery = new BoolQuery.Builder();
for (ProcessRole role : user.getProcessRoles()) {
negativeViewRoleQuery.should(termQuery("negativeViewRoles", role.getStringId())._toQuery());
}

TermsQueryField roleIds = new TermsQueryField.Builder()
.value(user.getProcessRoles().stream().map(ProcessRole::getStringId).map(FieldValue::of).collect(Collectors.toList()))
.build();

negativeViewRoleQuery.should(QueryBuilders.terms(term -> term.field("negativeViewRoles").terms(roleIds)));
negativeViewRole.mustNot(negativeViewRoleQuery.build()._toQuery());
return negativeViewRole.build();
}

private BoolQuery buildPositiveViewUser(BoolQuery viewPermNotExists, LoggedUser user) {
BoolQuery.Builder positiveViewUser = new BoolQuery.Builder();
BoolQuery.Builder positiveViewUserQuery = new BoolQuery.Builder();
positiveViewUserQuery.must(termQuery("viewUsers", user.getStringId())._toQuery());
positiveViewUser.should(viewPermNotExists._toQuery());
positiveViewUser.should(positiveViewUserQuery.build()._toQuery());
return positiveViewUser.build();
return new BoolQuery.Builder()
.should(viewPermNotExists._toQuery())
.filter(termQuery("viewUsers", user.getStringId())._toQuery())
.build();
}

private BoolQuery buildNegativeViewUser(LoggedUser user) {
BoolQuery.Builder negativeViewUser = new BoolQuery.Builder();
BoolQuery.Builder negativeViewUserQuery = new BoolQuery.Builder();
negativeViewUserQuery.should(termQuery("negativeViewUsers", user.getStringId())._toQuery());
negativeViewUser.mustNot(negativeViewUserQuery.build()._toQuery());
return negativeViewUser.build();
return new BoolQuery.Builder()
.mustNot(termQuery("negativeViewUsers", user.getStringId())._toQuery())
.build();
}

private BoolQuery setMinus(BoolQuery positiveSet, BoolQuery negativeSet) {
BoolQuery.Builder positiveSetMinusNegativeSet = new BoolQuery.Builder();
positiveSetMinusNegativeSet.must(positiveSet._toQuery());
positiveSetMinusNegativeSet.must(negativeSet._toQuery());
return positiveSetMinusNegativeSet.build();
return new BoolQuery.Builder()
.must(positiveSet._toQuery())
.must(negativeSet._toQuery())
.build();
}

private BoolQuery union(BoolQuery setA, BoolQuery setB) {
BoolQuery.Builder unionSet = new BoolQuery.Builder();
unionSet.should(setA._toQuery());
unionSet.should(setB._toQuery());
return unionSet.build();
return new BoolQuery.Builder()
.should(setA._toQuery())
.should(setB._toQuery())
.minimumShouldMatch(String.valueOf(1))
.build();
}
}
18 changes: 6 additions & 12 deletions ...gine/src/main/java/com/netgrif/application/engine/workflow/service/CaseSearchService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,10 +142,8 @@ public Predicate negativeViewUserQuery(String userId) {
public Predicate petriNet(Object query, LoggedUser user, Locale locale) {
List<PetriNetReference> allowedNets = petriNetService.getReferencesByUsersProcessRoles(user, locale);
if (query instanceof ArrayList) {
BooleanBuilder builder = new BooleanBuilder();
List<BooleanExpression> expressions = (List<BooleanExpression>) ((ArrayList) query).stream().filter(q -> q instanceof HashMap).map(q -> petriNetObject((HashMap<String, String>) q, allowedNets)).collect(Collectors.toList());
expressions.forEach(builder::or);
return builder;
List<Predicate> expressions = (List<Predicate>) ((ArrayList) query).stream().filter(q -> q instanceof HashMap).map(q -> petriNetObject((HashMap<String, String>) q, allowedNets)).collect(Collectors.toList());
return constructPredicateTree(expressions, BooleanBuilder::or);
} else if (query instanceof HashMap) {
return petriNetObject((HashMap<String, String>) query, allowedNets);
}
Expand Down Expand Up @@ -346,10 +344,8 @@ private String parseProcessIdentifier(HashMap<String, String> petriNet) {

public Predicate caseId(Object query) {
if (query instanceof ArrayList) {
BooleanBuilder builder = new BooleanBuilder();
List<BooleanExpression> expressions = (List<BooleanExpression>) ((ArrayList) query).stream().filter(q -> q instanceof String).map(q -> caseIdString((String) q)).collect(Collectors.toList());
expressions.forEach(builder::or);
return builder;
List<Predicate> expressions = (List<Predicate>) ((ArrayList) query).stream().filter(q -> q instanceof String).map(q -> caseIdString((String) q)).collect(Collectors.toList());
return constructPredicateTree(expressions, BooleanBuilder::or);
} else if (query instanceof String) {
return caseIdString((String) query);
}
Expand All @@ -371,9 +367,7 @@ public Predicate group(Object query, LoggedUser user, Locale locale) {
if (groupProcesses.size() == 0)
return null;

List<BooleanExpression> processQueries = groupProcesses.stream().map(PetriNetReference::getIdentifier).map(QCase.case$.processIdentifier::eq).collect(Collectors.toList());
BooleanBuilder builder = new BooleanBuilder();
processQueries.forEach(builder::or);
return builder;
List<Predicate> processQueries = groupProcesses.stream().map(PetriNetReference::getIdentifier).map(QCase.case$.processIdentifier::eq).collect(Collectors.toList());
return constructPredicateTree(processQueries, BooleanBuilder::or);
}
}
Loading
Loading