From d6079bf48e8bfe35e8f548252e507ffbc82330c1 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Mon, 16 Jun 2025 08:00:18 +0200 Subject: [PATCH 01/24] Introduced explicit index resolution API Signed-off-by: Nils Bandener --- .../ReindexFromRemoteWithAuthTests.java | 2 + .../plugin/wlm/AutoTaggingActionFilter.java | 2 + .../indices/alias/IndicesAliasesRequest.java | 3 +- .../alias/TransportIndicesAliasesAction.java | 128 +++++--- .../indices/create/AutoCreateAction.java | 19 +- .../create/TransportCreateIndexAction.java | 16 +- .../datastream/CreateDataStreamAction.java | 11 +- .../datastream/GetDataStreamAction.java | 15 +- .../delete/TransportDeleteIndexAction.java | 25 +- .../indices/TransportIndicesExistsAction.java | 12 +- .../put/TransportAutoPutMappingAction.java | 10 +- .../put/TransportPutMappingAction.java | 11 +- .../open/TransportOpenIndexAction.java | 17 +- .../rollover/TransportRolloverAction.java | 30 +- .../segments/TransportPitSegmentsAction.java | 16 +- .../get/TransportGetSettingsAction.java | 17 +- .../put/TransportUpdateSettingsAction.java | 17 +- .../indices/shrink/TransportResizeAction.java | 23 +- .../pause/TransportPauseIngestionAction.java | 17 +- .../action/bulk/BulkShardRequest.java | 4 + .../action/bulk/TransportBulkAction.java | 2 +- .../action/bulk/TransportShardBulkAction.java | 11 +- .../TransportSingleItemBulkWriteAction.java | 18 +- .../action/delete/TransportDeleteAction.java | 10 +- .../TransportFieldCapabilitiesAction.java | 49 +++- .../action/index/TransportIndexAction.java | 10 +- .../opensearch/action/search/PitService.java | 15 +- .../search/TransportDeletePitAction.java | 21 +- .../action/search/TransportSearchAction.java | 60 +++- .../action/support/ActionFilter.java | 2 + .../action/support/ActionRequestMetadata.java | 78 +++++ .../action/support/TransportAction.java | 2 +- .../TransportIndicesResolvingAction.java | 30 ++ .../broadcast/TransportBroadcastAction.java | 17 +- .../node/TransportBroadcastByNodeAction.java | 11 +- .../TransportBroadcastReplicationAction.java | 18 +- .../TransportReplicationAction.java | 9 +- ...ransportInstanceSingleOperationAction.java | 20 +- .../shard/TransportSingleShardAction.java | 26 +- .../cluster/metadata/ResolvedIndices.java | 275 ++++++++++++++++++ .../bulk/TransportBulkActionIngestTests.java | 3 +- .../TransportActionFilterChainTests.java | 1 + 42 files changed, 952 insertions(+), 131 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java create mode 100644 server/src/main/java/org/opensearch/action/support/TransportIndicesResolvingAction.java create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java index 51aa02c55070c..8cb63b91c44de 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/ReindexFromRemoteWithAuthTests.java @@ -39,6 +39,7 @@ import org.opensearch.action.search.SearchAction; import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilterChain; +import org.opensearch.action.support.ActionRequestMetadata; import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; @@ -233,6 +234,7 @@ public void app Task task, String action, Request request, + ActionRequestMetadata actionRequestMetadata, ActionListener listener, ActionFilterChain chain ) { diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/AutoTaggingActionFilter.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/AutoTaggingActionFilter.java index 112e2a11956f0..c6294ed7ac242 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/AutoTaggingActionFilter.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/AutoTaggingActionFilter.java @@ -13,6 +13,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilterChain; +import org.opensearch.action.support.ActionRequestMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.plugin.wlm.rule.attribute_extractor.IndicesExtractor; @@ -75,6 +76,7 @@ public void app Task task, String action, Request request, + ActionRequestMetadata actionRequestMetadata, ActionListener listener, ActionFilterChain chain ) { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java index dd915f8857162..32fc713c93047 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -35,6 +35,7 @@ import org.opensearch.OpenSearchGenerationException; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.AliasesRequest; +import org.opensearch.action.CompositeIndicesRequest; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.cluster.metadata.AliasAction; @@ -76,7 +77,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class IndicesAliasesRequest extends AcknowledgedRequest implements ToXContentObject { +public class IndicesAliasesRequest extends AcknowledgedRequest implements ToXContentObject, CompositeIndicesRequest { private List allAliasActions = new ArrayList<>(); private String origin = ""; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index a5e87482f7645..0ab19d8b1c63b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.RequestValidators; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; @@ -48,6 +49,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.MetadataIndexAliasesService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.regex.Regex; @@ -68,6 +70,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import static java.util.Collections.unmodifiableList; @@ -76,7 +79,9 @@ * * @opensearch.internal */ -public class TransportIndicesAliasesAction extends TransportClusterManagerNodeAction { +public class TransportIndicesAliasesAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportIndicesAliasesAction.class); @@ -131,45 +136,97 @@ protected void clusterManagerOperation( final IndicesAliasesRequest request, final ClusterState state, final ActionListener listener - ) { + ) throws Exception { // Expand the indices names List actions = request.aliasActions(); - List finalActions = new ArrayList<>(); + List finalActions = resolvedAliasActions(request, state, true); + if (finalActions.isEmpty() && false == actions.isEmpty()) { + throw new AliasesNotFoundException( + actions.stream().flatMap(a -> Arrays.stream(a.getOriginalAliases())).collect(Collectors.toSet()).toArray(new String[0]) + ); + } + request.aliasActions().clear(); + IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest(unmodifiableList(finalActions)) + .ackTimeout(request.timeout()) + .clusterManagerNodeTimeout(request.clusterManagerNodeTimeout()); + + indexAliasesService.indicesAliases(updateRequest, new ActionListener() { + @Override + public void onResponse(ClusterStateUpdateResponse response) { + listener.onResponse(new AcknowledgedResponse(response.isAcknowledged())); + } + + @Override + public void onFailure(Exception t) { + logger.debug("failed to perform aliases", t); + listener.onFailure(t); + } + }); + } + + @Override + public ResolvedIndices resolveIndices(IndicesAliasesRequest request) { + try { + Set indices = new HashSet<>(); + + for (AliasAction aliasAction : resolvedAliasActions(request, clusterService.state(), false)) { + if (aliasAction instanceof AliasAction.Add addAliasAction) { + indices.add(addAliasAction.getIndex()); + indices.add(addAliasAction.getAlias()); + } else if (aliasAction instanceof AliasAction.Remove removeAliasAction) { + indices.add(removeAliasAction.getIndex()); + indices.add(removeAliasAction.getAlias()); + } else if (aliasAction instanceof AliasAction.RemoveIndex removeIndexAction) { + // TODO special action + indices.add(removeIndexAction.getIndex()); + } + } + + return ResolvedIndices.of(indices); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + // This should not happen if validate=false is passed to resolvedAliasActions() + throw new RuntimeException(e); + } + } + + private List resolvedAliasActions(IndicesAliasesRequest request, ClusterState state, boolean validate) throws Exception { + List result = new ArrayList<>(); // Resolve all the AliasActions into AliasAction instances and gather all the aliases - Set aliases = new HashSet<>(); - for (IndicesAliasesRequest.AliasActions action : actions) { + for (IndicesAliasesRequest.AliasActions action : request.aliasActions()) { final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices( state, request.indicesOptions(), false, action.indices() ); - for (Index concreteIndex : concreteIndices) { - IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(concreteIndex.getName()); - assert indexAbstraction != null : "invalid cluster metadata. index [" + concreteIndex.getName() + "] was not found"; - if (indexAbstraction.getParentDataStream() != null) { - throw new IllegalArgumentException( - "The provided expressions [" - + String.join(",", action.indices()) - + "] match a backing index belonging to data stream [" - + indexAbstraction.getParentDataStream().getName() - + "]. Data streams and their backing indices don't support aliases." - ); + if (validate) { + for (Index concreteIndex : concreteIndices) { + IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(concreteIndex.getName()); + assert indexAbstraction != null : "invalid cluster metadata. index [" + concreteIndex.getName() + "] was not found"; + if (indexAbstraction.getParentDataStream() != null) { + throw new IllegalArgumentException( + "The provided expressions [" + + String.join(",", action.indices()) + + "] match a backing index belonging to data stream [" + + indexAbstraction.getParentDataStream().getName() + + "]. Data streams and their backing indices don't support aliases." + ); + } + } + final Optional maybeException = requestValidators.validateRequest(request, state, concreteIndices); + if (maybeException.isPresent()) { + throw maybeException.get(); } - } - final Optional maybeException = requestValidators.validateRequest(request, state, concreteIndices); - if (maybeException.isPresent()) { - listener.onFailure(maybeException.get()); - return; } - Collections.addAll(aliases, action.getOriginalAliases()); for (final Index index : concreteIndices) { switch (action.actionType()) { case ADD: for (String alias : concreteAliases(action, state.metadata(), index.getName())) { - finalActions.add( + result.add( new AliasAction.Add( index.getName(), alias, @@ -184,37 +241,19 @@ protected void clusterManagerOperation( break; case REMOVE: for (String alias : concreteAliases(action, state.metadata(), index.getName())) { - finalActions.add(new AliasAction.Remove(index.getName(), alias, action.mustExist())); + result.add(new AliasAction.Remove(index.getName(), alias, action.mustExist())); } break; case REMOVE_INDEX: - finalActions.add(new AliasAction.RemoveIndex(index.getName())); + result.add(new AliasAction.RemoveIndex(index.getName())); break; default: throw new IllegalArgumentException("Unsupported action [" + action.actionType() + "]"); } } } - if (finalActions.isEmpty() && false == actions.isEmpty()) { - throw new AliasesNotFoundException(aliases.toArray(new String[0])); - } - request.aliasActions().clear(); - IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest(unmodifiableList(finalActions)) - .ackTimeout(request.timeout()) - .clusterManagerNodeTimeout(request.clusterManagerNodeTimeout()); - indexAliasesService.indicesAliases(updateRequest, new ActionListener() { - @Override - public void onResponse(ClusterStateUpdateResponse response) { - listener.onResponse(new AcknowledgedResponse(response.isAcknowledged())); - } - - @Override - public void onFailure(Exception t) { - logger.debug("failed to perform aliases", t); - listener.onFailure(t); - } - }); + return result; } private static String[] concreteAliases(IndicesAliasesRequest.AliasActions action, Metadata metadata, String concreteIndex) { @@ -255,4 +294,5 @@ private static String[] concreteAliases(IndicesAliasesRequest.AliasActions actio return action.aliases(); } } + } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java index 4638c420ddb4d..efa84bdca660a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java @@ -35,6 +35,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.ActiveShardsObserver; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterState; @@ -49,6 +50,7 @@ import org.opensearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.opensearch.cluster.metadata.MetadataCreateIndexService; import org.opensearch.cluster.metadata.MetadataIndexTemplateService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; @@ -82,7 +84,9 @@ private AutoCreateAction() { * * @opensearch.internal */ - public static final class TransportAction extends TransportClusterManagerNodeAction { + public static final class TransportAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private final ActiveShardsObserver activeShardsObserver; private final MetadataCreateIndexService createIndexService; @@ -187,6 +191,19 @@ public ClusterState execute(ClusterState currentState) throws Exception { protected ClusterBlockException checkBlock(CreateIndexRequest request, ClusterState state) { return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.index()); } + + @Override + public ResolvedIndices resolveIndices(CreateIndexRequest request) { + ClusterState clusterState = clusterService.state(); + DataStreamTemplate dataStreamTemplate = resolveAutoCreateDataStream(request, clusterState.metadata()); + + if (dataStreamTemplate != null) { + // No date math is supported when a data stream is created + return ResolvedIndices.of(request.index()); + } else { + return ResolvedIndices.of(indexNameExpressionResolver.resolveDateMathExpression(request.index())); + } + } } static DataStreamTemplate resolveAutoCreateDataStream(CreateIndexRequest request, Metadata metadata) { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java index c1f7c6ef87e8a..31d3c1373b3c4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -33,12 +33,14 @@ package org.opensearch.action.admin.indices.create; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataCreateIndexService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -54,7 +56,9 @@ * * @opensearch.internal */ -public class TransportCreateIndexAction extends TransportClusterManagerNodeAction { +public class TransportCreateIndexAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private final MetadataCreateIndexService createIndexService; private final MappingTransformerRegistry mappingTransformerRegistry; @@ -115,7 +119,7 @@ protected void clusterManagerOperation( cause = "api"; } - final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index()); + final String indexName = resolveIndexName(request); final String finalCause = cause; final ActionListener mappingTransformListener = ActionListener.wrap(transformedMappings -> { @@ -143,4 +147,12 @@ protected void clusterManagerOperation( mappingTransformerRegistry.applyTransformers(request.mappings(), null, mappingTransformListener); } + @Override + public ResolvedIndices resolveIndices(CreateIndexRequest request) { + return ResolvedIndices.of(resolveIndexName(request)); + } + + private String resolveIndexName(CreateIndexRequest request) { + return indexNameExpressionResolver.resolveDateMathExpression(request.index()); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java index 6b5091e0b2ab5..70bcf2d015b98 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java @@ -37,6 +37,7 @@ import org.opensearch.action.ValidateActions; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; @@ -46,6 +47,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataCreateDataStreamService; import org.opensearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.inject.Inject; @@ -137,7 +139,9 @@ public IndicesOptions indicesOptions() { * * @opensearch.internal */ - public static class TransportAction extends TransportClusterManagerNodeAction { + public static class TransportAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private final MetadataCreateDataStreamService metadataCreateDataStreamService; @@ -179,6 +183,11 @@ protected void clusterManagerOperation(Request request, ClusterState state, Acti protected ClusterBlockException checkBlock(Request request, ClusterState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } + + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(request.name); + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java index 1db4e85887c23..edefdfa2b3759 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java @@ -38,6 +38,7 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; import org.opensearch.cluster.AbstractDiffable; @@ -49,6 +50,7 @@ import org.opensearch.cluster.metadata.DataStream; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataIndexTemplateService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; @@ -292,7 +294,9 @@ public int hashCode() { * * @opensearch.internal */ - public static class TransportAction extends TransportClusterManagerNodeReadAction { + public static class TransportAction extends TransportClusterManagerNodeReadAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportAction.class); @@ -354,6 +358,15 @@ static List getDataStreams(ClusterState clusterState, IndexNameExpre protected ClusterBlockException checkBlock(Request request, ClusterState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } + + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of( + getDataStreams(clusterService.state(), indexNameExpressionResolver, request).stream() + .map(DataStream::getName) + .collect(Collectors.toList()) + ); + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java index d6fc5386aed92..3641c6b06f34f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DestructiveOperations; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; @@ -44,6 +45,7 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataDeleteIndexService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -55,15 +57,15 @@ import java.io.IOException; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; /** * Delete index action. * * @opensearch.internal */ -public class TransportDeleteIndexAction extends TransportClusterManagerNodeAction { +public class TransportDeleteIndexAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportDeleteIndexAction.class); @@ -120,15 +122,15 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) { - final Set concreteIndices = new HashSet<>(Arrays.asList(indexNameExpressionResolver.concreteIndices(state, request))); - if (concreteIndices.isEmpty()) { + Index[] concreteIndices = resolveIndicesAsArray(request, state); + if (concreteIndices.length == 0) { listener.onResponse(new AcknowledgedResponse(true)); return; } DeleteIndexClusterStateUpdateRequest deleteRequest = new DeleteIndexClusterStateUpdateRequest().ackTimeout(request.timeout()) .clusterManagerNodeTimeout(request.clusterManagerNodeTimeout()) - .indices(concreteIndices.toArray(new Index[0])); + .indices(concreteIndices); deleteIndexService.deleteIndices(deleteRequest, new ActionListener() { @@ -139,9 +141,18 @@ public void onResponse(ClusterStateUpdateResponse response) { @Override public void onFailure(Exception t) { - logger.debug(() -> new ParameterizedMessage("failed to delete indices [{}]", concreteIndices), t); + logger.debug(() -> new ParameterizedMessage("failed to delete indices [{}]", Arrays.asList(concreteIndices)), t); listener.onFailure(t); } }); } + + @Override + public ResolvedIndices resolveIndices(DeleteIndexRequest request) { + return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + } + + private Index[] resolveIndicesAsArray(DeleteIndexRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteIndices(clusterState, request); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java index a298eae1aa865..7dc26641780f3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java @@ -34,11 +34,13 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -54,7 +56,9 @@ * * @opensearch.internal */ -public class TransportIndicesExistsAction extends TransportClusterManagerNodeReadAction { +public class TransportIndicesExistsAction extends TransportClusterManagerNodeReadAction + implements + TransportIndicesResolvingAction { @Inject public TransportIndicesExistsAction( @@ -119,4 +123,10 @@ protected void clusterManagerOperation( } listener.onResponse(new IndicesExistsResponse(exists)); } + + @Override + public ResolvedIndices resolveIndices(IndicesExistsRequest request) { + // TODO this is likely not correct + return ResolvedIndices.of(indexNameExpressionResolver.resolveExpressions(clusterService.state(), request.indices())); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java index f50acb6e2d56e..f87f26afc28c0 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.indices.mapping.put; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; @@ -39,6 +40,7 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataMappingService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -55,7 +57,9 @@ * * @opensearch.internal */ -public class TransportAutoPutMappingAction extends TransportClusterManagerNodeAction { +public class TransportAutoPutMappingAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private final MetadataMappingService metadataMappingService; @@ -116,4 +120,8 @@ protected void clusterManagerOperation( TransportPutMappingAction.performMappingUpdate(concreteIndices, request, listener, metadataMappingService); } + @Override + public ResolvedIndices resolveIndices(PutMappingRequest request) { + return ResolvedIndices.of(request.getConcreteIndex()); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index 4c37592714185..a156aeeca8e1f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.RequestValidators; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; @@ -45,6 +46,7 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataMappingService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -68,7 +70,9 @@ * * @opensearch.internal */ -public class TransportPutMappingAction extends TransportClusterManagerNodeAction { +public class TransportPutMappingAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportPutMappingAction.class); @@ -150,6 +154,11 @@ protected void clusterManagerOperation( } } + @Override + public ResolvedIndices resolveIndices(PutMappingRequest request) { + return ResolvedIndices.of(resolveIndices(clusterService.state(), request, indexNameExpressionResolver)); + } + static Index[] resolveIndices(final ClusterState state, PutMappingRequest request, final IndexNameExpressionResolver iner) { if (request.getConcreteIndex() == null) { if (request.writeIndexOnly()) { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java index 7bd21925eb11d..a8ecc62fa112c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DestructiveOperations; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ack.OpenIndexClusterStateUpdateResponse; @@ -44,6 +45,7 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataIndexStateService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -60,7 +62,9 @@ * * @opensearch.internal */ -public class TransportOpenIndexAction extends TransportClusterManagerNodeAction { +public class TransportOpenIndexAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportOpenIndexAction.class); @@ -119,7 +123,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) { - final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + final Index[] concreteIndices = resolveIndicesAsArray(request, state); if (concreteIndices == null || concreteIndices.length == 0) { listener.onResponse(new OpenIndexResponse(true, true)); return; @@ -143,4 +147,13 @@ public void onFailure(Exception t) { } }); } + + @Override + public ResolvedIndices resolveIndices(OpenIndexRequest request) { + return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + } + + private Index[] resolveIndicesAsArray(OpenIndexRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteIndices(clusterState, request); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java index 3124484716706..8168c9aab98df 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -39,6 +39,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.ActiveShardsObserver; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; @@ -47,6 +48,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; @@ -77,7 +79,9 @@ * * @opensearch.internal */ -public class TransportRolloverAction extends TransportClusterManagerNodeAction { +public class TransportRolloverAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private final MetadataRolloverService rolloverService; private final ActiveShardsObserver activeShardsObserver; @@ -260,6 +264,29 @@ public void onFailure(Exception e) { }); } + @Override + public ResolvedIndices resolveIndices(RolloverRequest rolloverRequest) { + try { + MetadataRolloverService.RolloverResult preResult = rolloverService.rolloverClusterState( + clusterService.state(), + rolloverRequest.getRolloverTarget(), + rolloverRequest.getNewIndexName(), + rolloverRequest.getCreateIndexRequest(), + Collections.emptyList(), + true, + true + ); + + return ResolvedIndices.of(preResult.sourceIndexName, preResult.rolloverIndexName); + } catch (Exception e) { + // Exceptions are mostly occurring due to validation errors (e.g. non-existing indices). + // These are not propagated to the caller because it should be still + // the clusterManagerOperation() that should report these failures. + // Instead, we return a basic result which still allows privilege evaluation. + return ResolvedIndices.ofNonNull(rolloverRequest.getNewIndexName(), rolloverRequest.getRolloverTarget()); + } + } + static Map evaluateConditions( final Collection> conditions, @Nullable final DocsStats docsStats, @@ -291,4 +318,5 @@ static Map evaluateConditions( return evaluateConditions(conditions, docsStats, metadata); } } + } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java index 393c07ba58c5e..293df3d3993cd 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.routing.PlainShardsIterator; import org.opensearch.cluster.routing.RecoverySource; @@ -96,7 +97,7 @@ public TransportPitSegmentsAction( */ @Override protected void doExecute(Task task, PitSegmentsRequest request, ActionListener listener) { - if (request.getPitIds().size() == 1 && "_all".equals(request.getPitIds().get(0))) { + if (isAllPitsRequest(request)) { pitService.getAllPits(ActionListener.wrap(response -> { request.clearAndSetPitIds(response.getPitInfos().stream().map(ListPitInfo::getPitId).collect(Collectors.toList())); super.doExecute(task, request, listener); @@ -106,6 +107,19 @@ protected void doExecute(Task task, PitSegmentsRequest request, ActionListener { +public class TransportGetSettingsAction extends TransportClusterManagerNodeReadAction + implements + TransportIndicesResolvingAction { private final SettingsFilter settingsFilter; private final IndexScopedSettings indexScopedSettings; @@ -112,7 +116,7 @@ private static boolean isFilteredRequest(GetSettingsRequest request) { @Override protected void clusterManagerOperation(GetSettingsRequest request, ClusterState state, ActionListener listener) { - Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + Index[] concreteIndices = resolveIndicesAsArray(request, state); final Map indexToSettingsBuilder = new HashMap<>(); final Map indexToDefaultSettingsBuilder = new HashMap<>(); for (Index concreteIndex : concreteIndices) { @@ -141,4 +145,13 @@ protected void clusterManagerOperation(GetSettingsRequest request, ClusterState } listener.onResponse(new GetSettingsResponse(indexToSettingsBuilder, indexToDefaultSettingsBuilder)); } + + @Override + public ResolvedIndices resolveIndices(GetSettingsRequest request) { + return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + } + + private Index[] resolveIndicesAsArray(GetSettingsRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteIndices(clusterState, request); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index fe1b139358b30..91670ac1d15c9 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; @@ -45,6 +46,7 @@ import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataUpdateSettingsService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -63,7 +65,9 @@ * * @opensearch.internal */ -public class TransportUpdateSettingsAction extends TransportClusterManagerNodeAction { +public class TransportUpdateSettingsAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportUpdateSettingsAction.class); @@ -158,7 +162,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) { - final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + final Index[] concreteIndices = resolveIndicesAsArray(request, state); UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest().indices( concreteIndices ) @@ -180,4 +184,13 @@ public void onFailure(Exception t) { } }); } + + @Override + public ResolvedIndices resolveIndices(UpdateSettingsRequest request) { + return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + } + + private Index[] resolveIndicesAsArray(UpdateSettingsRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteIndices(clusterState, request); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java index 0927ec234928a..7bf6d6b023eab 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java @@ -37,6 +37,7 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.stats.IndexShardStats; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -44,6 +45,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataCreateIndexService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; @@ -79,7 +81,9 @@ * * @opensearch.internal */ -public class TransportResizeAction extends TransportClusterManagerNodeAction { +public class TransportResizeAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private final MetadataCreateIndexService createIndexService; private final Client client; @@ -144,8 +148,8 @@ protected void clusterManagerOperation( ) { // there is no need to fetch docs stats for split but we keep it simple and do it anyway for simplicity of the code - final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex()); - final String targetIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index()); + final String sourceIndex = resolveSourceIndex(resizeRequest); + final String targetIndex = resolveTargetIndex(resizeRequest); IndexMetadata indexMetadata = state.metadata().index(sourceIndex); if (indexMetadata.getSettings().getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false) == true) { throw new IllegalStateException("cannot resize warm index"); @@ -227,6 +231,11 @@ protected void clusterManagerOperation( } + @Override + public ResolvedIndices resolveIndices(ResizeRequest resizeRequest) { + return ResolvedIndices.of(resolveSourceIndex(resizeRequest), resolveTargetIndex(resizeRequest)); + } + // static for unittesting this method static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest( final ResizeRequest resizeRequest, @@ -415,4 +424,12 @@ private static void validateRemoteMigrationModeSettings( } } } + + private String resolveSourceIndex(ResizeRequest resizeRequest) { + return indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex()); + } + + private String resolveTargetIndex(ResizeRequest resizeRequest) { + return indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index()); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/pause/TransportPauseIngestionAction.java b/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/pause/TransportPauseIngestionAction.java index aacf67974e5fa..6f2a2577c84a3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/pause/TransportPauseIngestionAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/pause/TransportPauseIngestionAction.java @@ -15,12 +15,14 @@ import org.opensearch.action.admin.indices.streamingingestion.state.UpdateIngestionStateResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DestructiveOperations; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataStreamingIngestionStateService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -38,7 +40,9 @@ * * @opensearch.experimental */ -public class TransportPauseIngestionAction extends TransportClusterManagerNodeAction { +public class TransportPauseIngestionAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportPauseIngestionAction.class); @@ -106,7 +110,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) throws Exception { - final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + final Index[] concreteIndices = resolveIndicesAsArray(request, state); if (concreteIndices == null || concreteIndices.length == 0) { listener.onResponse(new PauseIngestionResponse(true, false, new IngestionStateShardFailure[0], "")); return; @@ -144,4 +148,13 @@ public void onFailure(Exception e) { } ); } + + @Override + public ResolvedIndices resolveIndices(PauseIngestionRequest request) { + return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + } + + private Index[] resolveIndicesAsArray(PauseIngestionRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteIndices(clusterState, request); + } } diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkShardRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkShardRequest.java index c86f8c87eee15..453d98ea6255f 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkShardRequest.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkShardRequest.java @@ -74,6 +74,10 @@ public BulkItemRequest[] items() { @Override public String[] indices() { + // TODO: The following comment was possibly true on Elasticsearch, but it is not + // true on OpenSearch. Authorization also works with index names if an alias + // grants privileges for that particular index name. + // Thus, question: Shall we change this? // A bulk shard request encapsulates items targeted at a specific shard of an index. // However, items could be targeting aliases of the index, so the bulk request although // targeting a single concrete index shard might do so using several alias names. diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java index d0972e1276b39..79d523c6774bc 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java @@ -924,7 +924,7 @@ void executeBulk( * * @opensearch.internal */ - private static class ConcreteIndices { + static class ConcreteIndices { private final ClusterState state; private final IndexNameExpressionResolver indexNameExpressionResolver; private final Map indices = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java index dce69990e5724..7490eb9f6afa7 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java @@ -47,6 +47,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.ChannelActionListener; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.replication.ReplicationMode; import org.opensearch.action.support.replication.ReplicationOperation; @@ -62,6 +63,7 @@ import org.opensearch.cluster.action.shard.ShardStateAction; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.routing.ShardRouting; @@ -123,7 +125,9 @@ * * @opensearch.internal */ -public class TransportShardBulkAction extends TransportWriteAction { +public class TransportShardBulkAction extends TransportWriteAction + implements + TransportIndicesResolvingAction { public static final String ACTION_NAME = BulkAction.NAME + "[s]"; @@ -218,6 +222,11 @@ protected void handlePrimaryTermValidationRequest( } } + @Override + public ResolvedIndices resolveIndices(BulkShardRequest request) { + return ResolvedIndices.of(request.shardId().getIndexName()); + } + /** * This action is the primary term validation action which is used for doing primary term validation with replicas. * This is only applicable for TransportShardBulkAction because all writes (delete/update/single write/bulk) diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportSingleItemBulkWriteAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportSingleItemBulkWriteAction.java index 9b901dda24c2b..5b80b507b90e7 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportSingleItemBulkWriteAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportSingleItemBulkWriteAction.java @@ -36,10 +36,13 @@ import org.opensearch.action.DocWriteResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.WriteResponse; import org.opensearch.action.support.replication.ReplicatedWriteRequest; import org.opensearch.action.support.replication.ReplicationResponse; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.tasks.Task; @@ -53,19 +56,24 @@ @Deprecated public abstract class TransportSingleItemBulkWriteAction< Request extends ReplicatedWriteRequest, - Response extends ReplicationResponse & WriteResponse> extends HandledTransportAction { + Response extends ReplicationResponse & WriteResponse> extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final TransportBulkAction bulkAction; + private final IndexNameExpressionResolver indexNameExpressionResolver; protected TransportSingleItemBulkWriteAction( String actionName, TransportService transportService, ActionFilters actionFilters, Writeable.Reader requestReader, - TransportBulkAction bulkAction + TransportBulkAction bulkAction, + IndexNameExpressionResolver indexNameExpressionResolver ) { super(actionName, transportService, actionFilters, requestReader); this.bulkAction = bulkAction; + this.indexNameExpressionResolver = indexNameExpressionResolver; } @Override @@ -73,6 +81,11 @@ protected void doExecute(Task task, final Request request, final ActionListener< bulkAction.execute(task, toSingleItemBulkRequest(request), wrapBulkResponse(listener)); } + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(indexNameExpressionResolver.resolveDateMathExpression(request.index())); + } + public static ActionListener wrapBulkResponse( ActionListener listener ) { @@ -97,4 +110,5 @@ public static BulkRequest toSingleItemBulkRequest(ReplicatedWriteRequest requ request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); return bulkRequest; } + } diff --git a/server/src/main/java/org/opensearch/action/delete/TransportDeleteAction.java b/server/src/main/java/org/opensearch/action/delete/TransportDeleteAction.java index 6cbabfec6d763..96f6fe5df94f1 100644 --- a/server/src/main/java/org/opensearch/action/delete/TransportDeleteAction.java +++ b/server/src/main/java/org/opensearch/action/delete/TransportDeleteAction.java @@ -35,6 +35,7 @@ import org.opensearch.action.bulk.TransportBulkAction; import org.opensearch.action.bulk.TransportSingleItemBulkWriteAction; import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.inject.Inject; import org.opensearch.transport.TransportService; @@ -49,7 +50,12 @@ public class TransportDeleteAction extends TransportSingleItemBulkWriteAction { @Inject - public TransportDeleteAction(TransportService transportService, ActionFilters actionFilters, TransportBulkAction bulkAction) { - super(DeleteAction.NAME, transportService, actionFilters, DeleteRequest::new, bulkAction); + public TransportDeleteAction( + TransportService transportService, + ActionFilters actionFilters, + TransportBulkAction bulkAction, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super(DeleteAction.NAME, transportService, actionFilters, DeleteRequest::new, bulkAction, indexNameExpressionResolver); } } diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 28328e4cfc415..bef037a479c0b 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -35,8 +35,10 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.util.concurrent.CountDown; @@ -63,7 +65,9 @@ * * @opensearch.internal */ -public class TransportFieldCapabilitiesAction extends HandledTransportAction { +public class TransportFieldCapabilitiesAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final ThreadPool threadPool; private final ClusterService clusterService; private final TransportFieldCapabilitiesIndexAction shardAction; @@ -91,21 +95,11 @@ public TransportFieldCapabilitiesAction( protected void doExecute(Task task, FieldCapabilitiesRequest request, final ActionListener listener) { // retrieve the initial timestamp in case the action is a cross cluster search long nowInMillis = request.nowInMillis() == null ? System.currentTimeMillis() : request.nowInMillis(); - final ClusterState clusterState = clusterService.state(); - final Map remoteClusterIndices = remoteClusterService.groupIndices( - request.indicesOptions(), - request.indices(), - idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState) - ); - final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); - final String[] concreteIndices; - if (localIndices == null) { - // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices - concreteIndices = Strings.EMPTY_ARRAY; - } else { - concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, localIndices); - } - final int totalNumRequest = concreteIndices.length + remoteClusterIndices.size(); + ResolvedIndices allResolvedIndices = resolveIndices(request, clusterService.state()); + Set concreteIndices = allResolvedIndices.local().names(); + Map remoteClusterIndices = allResolvedIndices.remote().asClusterToOriginalIndicesMap(); + OriginalIndices localIndices = allResolvedIndices.local().originalIndices(); + final int totalNumRequest = concreteIndices.size() + remoteClusterIndices.size(); final CountDown completionCounter = new CountDown(totalNumRequest); final List indexResponses = Collections.synchronizedList(new ArrayList<>()); final Runnable onResponse = () -> { @@ -171,6 +165,29 @@ public void onFailure(Exception e) { } } + @Override + public ResolvedIndices resolveIndices(FieldCapabilitiesRequest request) { + return resolveIndices(request, clusterService.state()); + } + + private ResolvedIndices resolveIndices(FieldCapabilitiesRequest request, ClusterState clusterState) { + final Map remoteClusterIndices = remoteClusterService.groupIndices( + request.indicesOptions(), + request.indices(), + idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState) + ); + final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + final String[] concreteIndices; + if (localIndices == null) { + // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices + concreteIndices = Strings.EMPTY_ARRAY; + } else { + concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, localIndices); + } + + return ResolvedIndices.of(concreteIndices).withLocalOriginalIndices(localIndices).withRemoteIndices(remoteClusterIndices); + } + private FieldCapabilitiesResponse merge(List indexResponses, boolean includeUnmapped) { String[] indices = indexResponses.stream().map(FieldCapabilitiesIndexResponse::getIndexName).sorted().toArray(String[]::new); final Map> responseMapBuilder = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/action/index/TransportIndexAction.java b/server/src/main/java/org/opensearch/action/index/TransportIndexAction.java index ce32840f6751b..dccf2f720934a 100644 --- a/server/src/main/java/org/opensearch/action/index/TransportIndexAction.java +++ b/server/src/main/java/org/opensearch/action/index/TransportIndexAction.java @@ -35,6 +35,7 @@ import org.opensearch.action.bulk.TransportBulkAction; import org.opensearch.action.bulk.TransportSingleItemBulkWriteAction; import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.inject.Inject; import org.opensearch.transport.TransportService; @@ -56,7 +57,12 @@ public class TransportIndexAction extends TransportSingleItemBulkWriteAction { @Inject - public TransportIndexAction(ActionFilters actionFilters, TransportService transportService, TransportBulkAction bulkAction) { - super(IndexAction.NAME, transportService, actionFilters, IndexRequest::new, bulkAction); + public TransportIndexAction( + ActionFilters actionFilters, + TransportService transportService, + TransportBulkAction bulkAction, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super(IndexAction.NAME, transportService, actionFilters, IndexRequest::new, bulkAction, indexNameExpressionResolver); } } diff --git a/server/src/main/java/org/opensearch/action/search/PitService.java b/server/src/main/java/org/opensearch/action/search/PitService.java index 88bd572ab8eaa..9ec896e559fd1 100644 --- a/server/src/main/java/org/opensearch/action/search/PitService.java +++ b/server/src/main/java/org/opensearch/action/search/PitService.java @@ -28,9 +28,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -157,7 +159,7 @@ public void onFailure(final Exception e) { } /** - * This method returns indices associated for each pit + * This method returns indices associated for each pit. The result will be a Map that maps pitIds to index names. */ public Map getIndicesForPits(List pitIds) { Map pitToIndicesMap = new HashMap<>(); @@ -167,6 +169,17 @@ public Map getIndicesForPits(List pitIds) { return pitToIndicesMap; } + /** + * This method returns indices associated for each pit. The result will be a Set of all index names. + */ + public Set getIndicesForPitsFlat(Collection pitIds) { + Set result = new HashSet<>(); + for (String pitId : pitIds) { + result.addAll(Arrays.asList(SearchContextId.decode(nodeClient.getNamedWriteableRegistry(), pitId).getActualIndices())); + } + return result; + } + /** * Get all active point in time contexts */ diff --git a/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java b/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java index b15a4b66e8870..006bfba7516a0 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportDeletePitAction.java @@ -10,6 +10,8 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -27,7 +29,9 @@ /** * Transport action for deleting point in time searches - supports deleting list and all point in time searches */ -public class TransportDeletePitAction extends HandledTransportAction { +public class TransportDeletePitAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final NamedWriteableRegistry namedWriteableRegistry; private final PitService pitService; @@ -48,13 +52,26 @@ public TransportDeletePitAction( */ @Override protected void doExecute(Task task, DeletePitRequest request, ActionListener listener) { - if (request.getPitIds().size() == 1 && "_all".equals(request.getPitIds().get(0))) { + if (isAllPitsRequest(request)) { deleteAllPits(listener); } else { deletePits(listener, request); } } + @Override + public ResolvedIndices resolveIndices(DeletePitRequest request) { + if (isAllPitsRequest(request)) { + return ResolvedIndices.all(); + } else { + return ResolvedIndices.of(this.pitService.getIndicesForPitsFlat(request.getPitIds())); + } + } + + private boolean isAllPitsRequest(DeletePitRequest request) { + return request.getPitIds().size() == 1 && "_all".equals(request.getPitIds().get(0)); + } + /** * Deletes one or more point in time search contexts. */ diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index 541ed989b35a8..fa5b84c4940bb 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -41,11 +41,13 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.TimeoutTaskCancellationUtility; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.GroupShardsIterator; @@ -135,7 +137,9 @@ * * @opensearch.internal */ -public class TransportSearchAction extends HandledTransportAction { +public class TransportSearchAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { /** The maximum number of shards for a single search request. */ public static final Setting SHARD_COUNT_LIMIT_SETTING = Setting.longSetting( @@ -505,20 +509,10 @@ private ActionListener buildRewriteListener( searchRequest.source(source); } final ClusterState clusterState = clusterService.state(); - final SearchContextId searchContext; - final Map remoteClusterIndices; - if (searchRequest.pointInTimeBuilder() != null) { - searchContext = SearchContextId.decode(namedWriteableRegistry, searchRequest.pointInTimeBuilder().getId()); - remoteClusterIndices = getIndicesFromSearchContexts(searchContext, searchRequest.indicesOptions()); - } else { - searchContext = null; - remoteClusterIndices = remoteClusterService.groupIndices( - searchRequest.indicesOptions(), - searchRequest.indices(), - idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState) - ); - } - OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + final OriginalIndicesAndSearchContextId requestedIndices = extractRequestedIndices(searchRequest, clusterState); + final SearchContextId searchContext = requestedIndices.searchContextId; + final Map remoteClusterIndices = requestedIndices.remoteClusterIndices; + OriginalIndices localIndices = requestedIndices.localOriginalIndices; if (remoteClusterIndices.isEmpty()) { executeLocalSearch( task, @@ -996,6 +990,37 @@ static List getRemoteShardsIteratorFromPointInTime( return remoteShardIterators; } + @Override + public ResolvedIndices resolveIndices(SearchRequest searchRequest) { + ClusterState clusterState = clusterService.state(); + OriginalIndicesAndSearchContextId requestedIndices = extractRequestedIndices(searchRequest, clusterState); + Index[] localConcreteIndices = resolveLocalIndices( + requestedIndices.localOriginalIndices, + clusterState, + new SearchTimeProvider(searchRequest.getOrCreateAbsoluteStartMillis(), System.nanoTime(), System::nanoTime) + ); + + return ResolvedIndices.of(localConcreteIndices).withRemoteIndices(requestedIndices.remoteClusterIndices); + } + + private OriginalIndicesAndSearchContextId extractRequestedIndices(SearchRequest searchRequest, ClusterState clusterState) { + final SearchContextId searchContext; + final Map remoteClusterIndices; + if (searchRequest.pointInTimeBuilder() != null) { + searchContext = SearchContextId.decode(namedWriteableRegistry, searchRequest.pointInTimeBuilder().getId()); + remoteClusterIndices = getIndicesFromSearchContexts(searchContext, searchRequest.indicesOptions()); + } else { + searchContext = null; + remoteClusterIndices = remoteClusterService.groupIndices( + searchRequest.indicesOptions(), + searchRequest.indices(), + idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState) + ); + } + OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + return new OriginalIndicesAndSearchContextId(localIndices, remoteClusterIndices, searchContext); + } + private Index[] resolveLocalIndices(OriginalIndices localIndices, ClusterState clusterState, SearchTimeProvider timeProvider) { if (localIndices == null) { return Index.EMPTY_ARRAY; // don't search on any local index (happens when only remote indices were specified) @@ -1473,4 +1498,9 @@ static List getLocalLocalShardsIteratorFromPointInTime( } return iterators; } + + record OriginalIndicesAndSearchContextId(OriginalIndices localOriginalIndices, Map remoteClusterIndices, + SearchContextId searchContextId) { + + } } diff --git a/server/src/main/java/org/opensearch/action/support/ActionFilter.java b/server/src/main/java/org/opensearch/action/support/ActionFilter.java index e936512004fd2..0c7f5a6e220a0 100644 --- a/server/src/main/java/org/opensearch/action/support/ActionFilter.java +++ b/server/src/main/java/org/opensearch/action/support/ActionFilter.java @@ -57,6 +57,7 @@ void apply( Task task, String action, Request request, + ActionRequestMetadata actionRequestMetadata, ActionListener listener, ActionFilterChain chain ); @@ -72,6 +73,7 @@ public final vo Task task, String action, Request request, + ActionRequestMetadata actionRequestMetadata, ActionListener listener, ActionFilterChain chain ) { diff --git a/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java b/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java new file mode 100644 index 0000000000000..20b183309a09d --- /dev/null +++ b/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.support; + +import org.opensearch.action.ActionRequest; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.core.action.ActionResponse; + +import java.util.Optional; + +/** + * This class can be used to provide metadata about action requests to ActionFilter implementations. + * At the moment, this class provides information about the requested indices of a request, but it can be + * extended to transport further metadata. + */ +public class ActionRequestMetadata { + + /** + * Returns an empty meta data object which will just report unknown results. + */ + public static ActionRequestMetadata empty() { + @SuppressWarnings("unchecked") + ActionRequestMetadata result = (ActionRequestMetadata) EMPTY; + return result; + } + + private static final ActionRequestMetadata EMPTY = new ActionRequestMetadata<>(null, null); + + private final TransportAction transportAction; + private final Request request; + + private ResolvedIndices resolvedIndices; + private boolean resolvedIndicesInitialized; + + ActionRequestMetadata(TransportAction transportAction, Request request) { + this.transportAction = transportAction; + this.request = request; + } + + /** + * If the current action request references indices, this method actually referenced indices. That means that any + * expressions or patterns will be resolved. + *

+ * If the request cannot reference indices OR if the respective action does not support resolving of requests, + * this returns an empty Optional. + */ + public Optional resolvedIndices() { + if (!(transportAction instanceof TransportIndicesResolvingAction)) { + return Optional.empty(); + } + + if (this.resolvedIndicesInitialized) { + return Optional.of(this.resolvedIndices); + } else { + return resolveIndices(); + } + } + + /** + * Performs the actual index resolution. Index resolution can be relatively costly on big clusters, so we + * perform it lazily only when requested. + */ + private Optional resolveIndices() { + @SuppressWarnings("unchecked") + TransportIndicesResolvingAction indicesResolvingAction = (TransportIndicesResolvingAction) this.transportAction; + ResolvedIndices result = indicesResolvingAction.resolveIndices(request); + + this.resolvedIndices = result; + this.resolvedIndicesInitialized = true; + return Optional.of(result); + } +} diff --git a/server/src/main/java/org/opensearch/action/support/TransportAction.java b/server/src/main/java/org/opensearch/action/support/TransportAction.java index f71347f6f1d07..3cc79a407bd29 100644 --- a/server/src/main/java/org/opensearch/action/support/TransportAction.java +++ b/server/src/main/java/org/opensearch/action/support/TransportAction.java @@ -215,7 +215,7 @@ public void proceed(Task task, String actionName, Request request, ActionListene int i = index.getAndIncrement(); try { if (i < this.action.filters.length) { - this.action.filters[i].apply(task, actionName, request, listener, this); + this.action.filters[i].apply(task, actionName, request, new ActionRequestMetadata<>(action, request), listener, this); } else if (i == this.action.filters.length) { this.action.doExecute(task, request, listener); } else { diff --git a/server/src/main/java/org/opensearch/action/support/TransportIndicesResolvingAction.java b/server/src/main/java/org/opensearch/action/support/TransportIndicesResolvingAction.java new file mode 100644 index 0000000000000..120117ade6466 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/support/TransportIndicesResolvingAction.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.support; + +import org.opensearch.action.ActionRequest; +import org.opensearch.cluster.metadata.ResolvedIndices; + +/** + * An additional interface that should be implemented by TransportAction implementations which need to resolve + * IndicesRequests or other action requests which specify indices. This interface allows other components to retrieve + * precise information about the indices an action is going to operate on. This is particularly useful for access + * control implementations, but can be also used for other purposes, such as monitoring, audit logging, etc. + *

+ * Classes implementing this interface should make sure that the reported indices are also actually the indices + * the action will operate on. The best way to achieve this, is to move the index extraction code from the execute + * methods into reusable methods and to depend on these both for execution and reporting. + */ +public interface TransportIndicesResolvingAction { + + /** + * Returns the actual indices the action will operate on, given the specified request and cluster state. + */ + ResolvedIndices resolveIndices(Request request); +} diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java index 8bf8555194976..06c137d0fb477 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java @@ -38,9 +38,11 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.TransportActions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.FailAwareWeightedRouting; @@ -73,7 +75,9 @@ public abstract class TransportBroadcastAction< Request extends BroadcastRequest, Response extends BroadcastResponse, ShardRequest extends BroadcastShardRequest, - ShardResponse extends BroadcastShardResponse> extends HandledTransportAction { + ShardResponse extends BroadcastShardResponse> extends HandledTransportAction + implements + TransportIndicesResolvingAction { protected final ClusterService clusterService; protected final TransportService transportService; @@ -107,6 +111,15 @@ protected void doExecute(Task task, Request request, ActionListener li new AsyncBroadcastAction(task, request, listener).start(); } + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(indexNameExpressionResolver.concreteIndexNames(clusterService.state(), request)); + } + + protected ResolvedIndices resolveIndices(Request request, ClusterState clusterState) { + return ResolvedIndices.of(indexNameExpressionResolver.concreteIndexNames(clusterState, request)); + } + protected abstract Response newResponse(Request request, AtomicReferenceArray shardsResponses, ClusterState clusterState); protected abstract ShardRequest newShardRequest(int numShards, ShardRouting shard, Request request); @@ -154,7 +167,7 @@ protected AsyncBroadcastAction(Task task, Request request, ActionListener, Response extends BroadcastResponse, - ShardOperationResult extends Writeable> extends HandledTransportAction { + ShardOperationResult extends Writeable> extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final ClusterService clusterService; private final TransportService transportService; @@ -280,6 +284,11 @@ protected void doExecute(Task task, Request request, ActionListener li new AsyncAction(task, request, listener).start(); } + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(resolveConcreteIndexNames(clusterService.state(), request)); + } + /** * Asynchronous action * diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java index e235adbc162fc..c13cffe598456 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java @@ -36,12 +36,14 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.TransportActions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.broadcast.BroadcastRequest; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.action.support.broadcast.BroadcastShardOperationFailedException; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.util.concurrent.CountDown; @@ -55,6 +57,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; /** @@ -67,7 +70,9 @@ public abstract class TransportBroadcastReplicationAction< Request extends BroadcastRequest, Response extends BroadcastResponse, ShardRequest extends ReplicationRequest, - ShardResponse extends ReplicationResponse> extends HandledTransportAction { + ShardResponse extends ReplicationResponse> extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final TransportReplicationAction replicatedBroadcastShardAction; private final ClusterService clusterService; @@ -138,6 +143,15 @@ public void onFailure(Exception e) { } } + @Override + public ResolvedIndices resolveIndices(Request request) { + return resolveIndices(request, clusterService.state()); + } + + private ResolvedIndices resolveIndices(Request request, ClusterState clusterState) { + return ResolvedIndices.of(indexNameExpressionResolver.concreteIndexNames(clusterState, request)); + } + protected void shardExecute(Task task, Request request, ShardId shardId, ActionListener shardActionListener) { ShardRequest shardRequest = newShardRequest(request, shardId); shardRequest.setParentTask(clusterService.localNode().getId(), task.getId()); @@ -149,7 +163,7 @@ protected void shardExecute(Task task, Request request, ShardId shardId, ActionL */ protected List shards(Request request, ClusterState clusterState) { List shardIds = new ArrayList<>(); - String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request); + Set concreteIndices = resolveIndices(request, clusterState).local().names(); for (String index : concreteIndices) { IndexMetadata indexMetadata = clusterState.metadata().getIndices().get(index); if (indexMetadata != null) { diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java index f474ce787992f..f6fe930e5ce20 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java @@ -43,6 +43,7 @@ import org.opensearch.action.support.ChannelActionListener; import org.opensearch.action.support.TransportAction; import org.opensearch.action.support.TransportActions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.replication.ReplicationOperation.Replicas; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateObserver; @@ -50,6 +51,7 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.routing.ShardRouting; @@ -111,7 +113,7 @@ public abstract class TransportReplicationAction< Request extends ReplicationRequest, ReplicaRequest extends ReplicationRequest, - Response extends ReplicationResponse> extends TransportAction { + Response extends ReplicationResponse> extends TransportAction implements TransportIndicesResolvingAction { /** * The timeout for retrying replication requests. @@ -318,6 +320,11 @@ protected void doExecute(Task task, Request request, ActionListener li runReroutePhase(task, request, listener, true); } + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(request.index); + } + private void runReroutePhase(Task task, Request request, ActionListener listener, boolean initiatedByNodeClient) { try { new ReroutePhase((ReplicationTask) task, request, listener, initiatedByNodeClient).run(); diff --git a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index 21d4ba726e86f..f9ef23c711180 100644 --- a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -35,11 +35,13 @@ import org.opensearch.action.UnavailableShardsException; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateObserver; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.routing.ShardRouting; @@ -76,7 +78,7 @@ */ public abstract class TransportInstanceSingleOperationAction< Request extends InstanceShardOperationRequest, - Response extends ActionResponse> extends HandledTransportAction { + Response extends ActionResponse> extends HandledTransportAction implements TransportIndicesResolvingAction { protected final ThreadPool threadPool; protected final ClusterService clusterService; @@ -108,6 +110,22 @@ protected void doExecute(Task task, Request request, ActionListener li new AsyncSingleAction(request, listener).start(); } + @Override + public ResolvedIndices resolveIndices(Request request) { + if (request.concreteIndex() != null) { + return ResolvedIndices.of(request.concreteIndex()); + } else { + try { + // TODO shall we possibly also set request.concreteIndex here? + return ResolvedIndices.of(indexNameExpressionResolver.concreteWriteIndex(clusterService.state(), request).getName()); + } catch (IndexNotFoundException e) { + // We just return the original unresolved expression. The error we encountered here will + // be encountered again in the doStart() method + return ResolvedIndices.of(request.index()); + } + } + } + protected abstract String executor(ShardId shardId); protected abstract void shardOperation(Request request, ActionListener listener); diff --git a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java index df91559a2f8cb..4a934002e9f81 100644 --- a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java @@ -39,10 +39,12 @@ import org.opensearch.action.support.ChannelActionListener; import org.opensearch.action.support.TransportAction; import org.opensearch.action.support.TransportActions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.FailAwareWeightedRouting; @@ -76,7 +78,9 @@ * @opensearch.internal */ public abstract class TransportSingleShardAction, Response extends ActionResponse> extends - TransportAction { + TransportAction + implements + TransportIndicesResolvingAction { protected final ThreadPool threadPool; protected final ClusterService clusterService; @@ -154,6 +158,19 @@ protected void resolveRequest(ClusterState state, InternalRequest request) { @Nullable protected abstract ShardsIterator shards(ClusterState state, InternalRequest request); + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(resolveToConcreteSingleIndex(request, clusterService.state())); + } + + private String resolveToConcreteSingleIndex(Request request, ClusterState clusterState) { + if (resolveIndex(request)) { + return indexNameExpressionResolver.concreteSingleIndex(clusterState, request).getName(); + } else { + return request.index(); + } + } + /** * Asynchronous single action * @@ -180,12 +197,7 @@ private AsyncSingleAction(Request request, ActionListener listener) { throw blockException; } - String concreteSingleIndex; - if (resolveIndex(request)) { - concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, request).getName(); - } else { - concreteSingleIndex = request.index(); - } + String concreteSingleIndex = resolveToConcreteSingleIndex(request, clusterState); this.internalRequest = new InternalRequest(request, concreteSingleIndex); resolveRequest(clusterState, internalRequest); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java new file mode 100644 index 0000000000000..2ec9f9795346a --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java @@ -0,0 +1,275 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.action.OriginalIndices; +import org.opensearch.cluster.ClusterState; +import org.opensearch.core.index.Index; +import org.opensearch.transport.RemoteClusterService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A class that encapsulates resolved indices. Resolved indices do not any wildcards or date math expressions. + * However, in contrast to the concept of "concrete indices", resolved indices might not exist yet, or might + * refer to aliases or data streams. + *

+ * ResolvedIndices classes are primarily created by the resolveIndices() method in TransportIndicesResolvingAction. + *

+ * Instances of ResolvedIndices are immutable. It will be not possible to modify the returned collection instances. + * All methods which add/modify elements will return a new ResolvedIndices instance. + *

+ * How resolved indices are obtained depends on the respective action and the associated requests: + *

    + *
  • If a request carries an index expression (i.e, might contain patterns or date math expressions), the index + * expression must be resolved using the appropriate index options; these might be request-specific or action-specific.
    • + *
  • Some requests already carry concrete indices; in these cases, the names of the concrete indices can be + * just taken without further evaluation
    • + *
+ */ +public class ResolvedIndices { + public static ResolvedIndices of(String... indices) { + return new ResolvedIndices( + new Local(Collections.unmodifiableSet(new HashSet<>(Arrays.asList(indices))), null, false), + Remote.EMPTY + ); + } + + public static ResolvedIndices of(Index... indices) { + return new ResolvedIndices( + new Local(Stream.of(indices).map(Index::getName).collect(Collectors.toUnmodifiableSet()), null, false), + Remote.EMPTY + ); + } + + public static ResolvedIndices of(Collection indices) { + return new ResolvedIndices(new Local(Collections.unmodifiableSet(new HashSet<>(indices)), null, false), Remote.EMPTY); + } + + public static ResolvedIndices all() { + return ALL; + } + + public static ResolvedIndices ofNonNull(String... indices) { + Set indexSet = new HashSet<>(indices.length); + + for (String index : indices) { + if (index != null) { + indexSet.add(index); + } + } + + return new ResolvedIndices(new Local(Collections.unmodifiableSet(indexSet), null, false), Remote.EMPTY); + } + + private static final ResolvedIndices ALL = new ResolvedIndices(new Local(Set.of(Metadata.ALL), null, true), Remote.EMPTY); + + private final Local local; + private final Remote remote; + + private ResolvedIndices(Local local, Remote remote) { + this.local = local; + this.remote = remote; + } + + public Local local() { + return this.local; + } + + public Remote remote() { + return this.remote; + } + + public ResolvedIndices withRemoteIndices(Map remoteIndices) { + if (remoteIndices.isEmpty()) { + return this; + } + + Map newRemoteIndices = new HashMap<>(remoteIndices); + newRemoteIndices.putAll(this.remote.clusterToOriginalIndicesMap); + + return new ResolvedIndices(this.local, new Remote(Collections.unmodifiableMap(newRemoteIndices))); + } + + public ResolvedIndices withLocalOriginalIndices(OriginalIndices originalIndices) { + return new ResolvedIndices(new Local(this.local.names, originalIndices, this.local.isAll), this.remote); + } + + public boolean isEmpty() { + return this.local.isEmpty() && this.remote.isEmpty(); + } + + /** + * Represents the local (i.e., non-remote) indices referenced by the respective request. + */ + public static class Local { + private final Set names; + private final OriginalIndices originalIndices; + private final boolean isAll; + + private Local(Set names, OriginalIndices originalIndices, boolean isAll) { + this.names = names; + this.originalIndices = originalIndices; + this.isAll = isAll; + } + + /** + * Returns all the local names. These names might be indices, aliases or data streams, depending on the usage. + *

+ * Note: You must gate this call by if (!isAll()). + * If isAll() is true, this method will throw an IllegalStateException. If you are sure that you really need all index names, please use the method + * names(ClusterState) instead. + * + * @return an unmodifiable set of names of indices, aliases and/or data streams. + * @throws IllegalStateException if isAll() is true + */ + public Set names() { + if (this.isAll) { + throw new IllegalStateException("ResolvedIndices.Local.names() cannot be called for isAll cases"); + } + + return this.names; + } + + /** + * Returns all the local names. These names might be indices, aliases or data streams, depending on the usage. + *

+ * Note: You must gate this call by if (!isAll()). + * If isAll() is true, this method will throw an IllegalStateException. If you are sure that you really need all index names, please use the method + * names(ClusterState) instead. + * + * @return an array of names of indices, aliases and/or data streams. + * @throws IllegalStateException if isAll() is true + */ + public String[] namesAsArray() { + return this.names().toArray(new String[0]); + } + + /** + * Returns all the local names. These names might be indices, aliases or data streams, depending on the usage. + *

+ * In case this is an isAll() object, this will return a set of all concrete indices on the cluster (incl. + * hidden and closed indices). This might be a large object. Be prepared to handle such a large object. + *

+ * This method will be only rarely needed. In most cases names() will be sufficient. + */ + public Set names(ClusterState clusterState) { + if (this.isAll) { + return clusterState.metadata().getIndicesLookup().keySet(); + } else { + return this.names; + } + } + + public OriginalIndices originalIndices() { + return this.originalIndices; + } + + public boolean isEmpty() { + if (this.isAll) { + return false; + } else { + return this.names.isEmpty(); + } + } + + public boolean isAll() { + return this.isAll; + } + + public boolean contains(String index) { + if (this.isAll) { + return true; + } else { + return this.names.contains(index); + } + } + + public boolean containsAny(Collection indices) { + if (this.isAll) { + return true; + } else { + return indices.stream().anyMatch(this.names::contains); + } + } + + public boolean containsAny(Predicate indexNamePredicate) { + return this.names.stream().anyMatch(indexNamePredicate); + } + } + + /** + * Represents the remote indices part of the respective request. + */ + public static class Remote { + static final Remote EMPTY = new Remote(Collections.emptyMap(), Collections.emptyList()); + + private final Map clusterToOriginalIndicesMap; + private List rawExpressions; + + private Remote(Map clusterToOriginalIndicesMap) { + this.clusterToOriginalIndicesMap = clusterToOriginalIndicesMap; + } + + private Remote(Map clusterToOriginalIndicesMap, List rawExpressions) { + this.clusterToOriginalIndicesMap = clusterToOriginalIndicesMap; + this.rawExpressions = rawExpressions; + } + + public Map asClusterToOriginalIndicesMap() { + return this.clusterToOriginalIndicesMap; + } + + public List asRawExpressions() { + List result = this.rawExpressions; + + if (result == null) { + result = this.rawExpressions = buildRawExpressions(); + } + + return result; + } + + public String[] asRawExpressionsArray() { + return this.asRawExpressions().toArray(new String[0]); + } + + public boolean isEmpty() { + return this.clusterToOriginalIndicesMap.isEmpty(); + } + + private List buildRawExpressions() { + if (this.clusterToOriginalIndicesMap.isEmpty()) { + return Collections.emptyList(); + } + + List result = new ArrayList<>(); + + for (Map.Entry entry : this.clusterToOriginalIndicesMap.entrySet()) { + for (String remoteIndex : entry.getValue().indices()) { + result.add(RemoteClusterService.buildRemoteIndexName(entry.getKey(), remoteIndex)); + } + } + + return Collections.unmodifiableList(result); + } + } + +} diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java index 40ecef9f3a51a..5511b67848c1e 100644 --- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java @@ -224,7 +224,8 @@ class TestSingleItemBulkWriteAction extends TransportSingleItemBulkWriteAction void app Task task, String action, Request request, + ActionRequestMetadata actionRequestMetadata, ActionListener listener, ActionFilterChain chain ) { From 4baa69f7c0445b4f09c45f02f6c9f4ba1dfd6ee3 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Mon, 16 Jun 2025 08:40:21 +0200 Subject: [PATCH 02/24] Fix Signed-off-by: Nils Bandener --- .../opensearch/plugin/wlm/AutoTaggingActionFilterTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/AutoTaggingActionFilterTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/AutoTaggingActionFilterTests.java index e5fb23af4a7e9..ed5e8e25843ea 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/AutoTaggingActionFilterTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/AutoTaggingActionFilterTests.java @@ -12,6 +12,7 @@ import org.opensearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.ActionFilterChain; +import org.opensearch.action.support.ActionRequestMetadata; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; @@ -77,7 +78,7 @@ public void testApplyForValidRequest() { when(request.indices()).thenReturn(new String[] { "foo" }); try (ThreadContext.StoredContext context = threadPool.getThreadContext().stashContext()) { when(ruleProcessingService.evaluateLabel(anyList())).thenReturn(Optional.of("TestQG_ID")); - autoTaggingActionFilter.apply(mock(Task.class), "Test", request, null, mockFilterChain); + autoTaggingActionFilter.apply(mock(Task.class), "Test", request, ActionRequestMetadata.empty(), null, mockFilterChain); assertEquals("TestQG_ID", threadPool.getThreadContext().getHeader(WorkloadGroupTask.WORKLOAD_GROUP_ID_HEADER)); verify(ruleProcessingService, times(1)).evaluateLabel(anyList()); @@ -87,7 +88,7 @@ public void testApplyForValidRequest() { public void testApplyForInValidRequest() { ActionFilterChain mockFilterChain = mock(TestActionFilterChain.class); CancelTasksRequest request = new CancelTasksRequest(); - autoTaggingActionFilter.apply(mock(Task.class), "Test", request, null, mockFilterChain); + autoTaggingActionFilter.apply(mock(Task.class), "Test", request, ActionRequestMetadata.empty(), null, mockFilterChain); verify(ruleProcessingService, times(0)).evaluateLabel(anyList()); } From dca0ccd007656f69eafdd2ba3c034d44b1a8a536 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 4 Jul 2025 11:40:50 +0200 Subject: [PATCH 03/24] Fix Signed-off-by: Nils Bandener --- .../opensearch/indices/SystemIndexRegistry.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java index ab2cbd4ef1a73..75bcf0b9f8d98 100644 --- a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java +++ b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java @@ -59,6 +59,10 @@ public static Set matchesSystemIndexPattern(Set indexExpressions return indexExpressions.stream().filter(pattern -> Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, pattern)).collect(Collectors.toSet()); } + public static boolean matchesSystemIndexPattern(String index) { + return Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, index); + } + public static Set matchesPluginSystemIndexPattern(String pluginClassName, Set indexExpressions) { if (!SYSTEM_INDEX_DESCRIPTORS_MAP.containsKey(pluginClassName)) { return Collections.emptySet(); @@ -72,6 +76,19 @@ public static Set matchesPluginSystemIndexPattern(String pluginClassName .collect(Collectors.toSet()); } + public static boolean matchesPluginSystemIndexPattern(String pluginClassName, String index) { + Collection systemIndexDescriptors = SYSTEM_INDEX_DESCRIPTORS_MAP.get(pluginClassName); + if (systemIndexDescriptors == null) { + return false; + } + String[] pluginSystemIndexPatterns = systemIndexDescriptors + .stream() + .map(SystemIndexDescriptor::getIndexPattern) + .toArray(String[]::new); + + + } + static List getAllDescriptors() { return SYSTEM_INDEX_DESCRIPTORS_MAP.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); } From 8053eabaae181ce7ff9d680f54b3a71a4ed3f282 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Thu, 28 Aug 2025 13:03:55 +0200 Subject: [PATCH 04/24] Introduced OptionalResolvedIndices class Signed-off-by: Nils Bandener --- .../TransportRenderSearchTemplateAction.java | 14 +- .../TransportSearchTemplateAction.java | 22 +- .../reindex/TransportDeleteByQueryAction.java | 17 +- .../reindex/TransportUpdateByQueryAction.java | 17 +- .../reindex/UpdateByQueryWithScriptTests.java | 1 + .../TransportClusterSearchShardsAction.java | 17 +- .../alias/TransportIndicesAliasesAction.java | 39 +- .../alias/get/TransportGetAliasesAction.java | 33 +- .../close/TransportCloseIndexAction.java | 17 +- .../datastream/DeleteDataStreamAction.java | 36 +- .../delete/TransportDeleteIndexAction.java | 8 +- .../indices/TransportIndicesExistsAction.java | 3 +- .../get/TransportGetFieldMappingsAction.java | 17 +- .../put/TransportPutMappingAction.java | 14 +- .../open/TransportOpenIndexAction.java | 8 +- .../TransportAddIndexBlockAction.java | 17 +- .../segments/TransportPitSegmentsAction.java | 5 +- .../get/TransportGetSettingsAction.java | 8 +- .../put/TransportUpdateSettingsAction.java | 8 +- .../TransportIndicesShardStoresAction.java | 17 +- .../pause/TransportPauseIngestionAction.java | 20 +- .../TransportResumeIngestionAction.java | 17 +- .../TransportHotToWarmTieringAction.java | 17 +- .../action/bulk/TransportShardBulkAction.java | 2 +- .../TransportFieldCapabilitiesAction.java | 7 +- ...TransportFieldCapabilitiesIndexAction.java | 9 +- .../action/search/CreatePitController.java | 11 +- .../action/search/CreatePitRequest.java | 9 + .../search/TransportCreatePitAction.java | 11 +- .../search/TransportDeletePitAction.java | 5 +- .../action/search/TransportSearchAction.java | 24 +- .../action/support/ActionRequestMetadata.java | 24 +- .../TransportIndicesResolvingAction.java | 4 +- .../broadcast/TransportBroadcastAction.java | 8 +- .../node/TransportBroadcastByNodeAction.java | 13 +- .../info/TransportClusterInfoAction.java | 19 +- .../TransportBroadcastReplicationAction.java | 8 +- ...ransportInstanceSingleOperationAction.java | 1 - .../metadata/IndexNameExpressionResolver.java | 210 ++++++++--- .../metadata/OptionallyResolvedIndices.java | 98 +++++ .../cluster/metadata/ResolvedIndices.java | 339 ++++++++++++++---- .../indices/SystemIndexRegistry.java | 8 +- .../IndexNameExpressionResolverTests.java | 112 +++++- .../metadata/ResolvedIndicesTests.java | 99 +++++ 44 files changed, 1117 insertions(+), 276 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java create mode 100644 server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportRenderSearchTemplateAction.java index abd5d768f4a11..28a411d010543 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportRenderSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportRenderSearchTemplateAction.java @@ -8,6 +8,7 @@ package org.opensearch.script.mustache; +import org.opensearch.action.search.TransportSearchAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.common.inject.Inject; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -23,8 +24,17 @@ public TransportRenderSearchTemplateAction( ActionFilters actionFilters, ScriptService scriptService, NamedXContentRegistry xContentRegistry, - NodeClient client + NodeClient client, + TransportSearchAction transportSearchAction ) { - super(RenderSearchTemplateAction.NAME, transportService, actionFilters, scriptService, xContentRegistry, client); + super( + RenderSearchTemplateAction.NAME, + transportService, + actionFilters, + scriptService, + xContentRegistry, + client, + transportSearchAction + ); } } diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java index e24f3c0d7d560..85cf7a606a08f 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java @@ -34,8 +34,11 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.TransportSearchAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.common.inject.Inject; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.action.ActionListener; @@ -57,13 +60,15 @@ import java.io.IOException; import java.util.Collections; -public class TransportSearchTemplateAction extends HandledTransportAction { - +public class TransportSearchTemplateAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private static final String TEMPLATE_LANG = MustacheScriptEngine.NAME; protected final ScriptService scriptService; protected final NamedXContentRegistry xContentRegistry; protected final NodeClient client; + private final TransportSearchAction transportSearchAction; @Inject public TransportSearchTemplateAction( @@ -71,12 +76,14 @@ public TransportSearchTemplateAction( ActionFilters actionFilters, ScriptService scriptService, NamedXContentRegistry xContentRegistry, - NodeClient client + NodeClient client, + TransportSearchAction transportSearchAction ) { super(SearchTemplateAction.NAME, transportService, actionFilters, SearchTemplateRequest::new); this.scriptService = scriptService; this.xContentRegistry = xContentRegistry; this.client = client; + this.transportSearchAction = transportSearchAction; } public TransportSearchTemplateAction( @@ -85,12 +92,14 @@ public TransportSearchTemplateAction( ActionFilters actionFilters, ScriptService scriptService, NamedXContentRegistry xContentRegistry, - NodeClient client + NodeClient client, + TransportSearchAction transportSearchAction ) { super(actionName, transportService, actionFilters, SearchTemplateRequest::new); this.scriptService = scriptService; this.xContentRegistry = xContentRegistry; this.client = client; + this.transportSearchAction = transportSearchAction; } @Override @@ -180,4 +189,9 @@ private static void checkRestTotalHitsAsInt(SearchRequest searchRequest, SearchS } } } + + @Override + public ResolvedIndices resolveIndices(SearchTemplateRequest request) { + return transportSearchAction.resolveIndices(request.getRequest()); + } } diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportDeleteByQueryAction.java index 76e3cc42697ff..986d1a32845fb 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportDeleteByQueryAction.java @@ -32,8 +32,11 @@ package org.opensearch.index.reindex; +import org.opensearch.action.search.TransportSearchAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -45,12 +48,15 @@ import org.opensearch.transport.client.Client; import org.opensearch.transport.client.ParentTaskAssigningClient; -public class TransportDeleteByQueryAction extends HandledTransportAction { +public class TransportDeleteByQueryAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final ThreadPool threadPool; private final Client client; private final ScriptService scriptService; private final ClusterService clusterService; + private final TransportSearchAction transportSearchAction; @Inject public TransportDeleteByQueryAction( @@ -59,7 +65,8 @@ public TransportDeleteByQueryAction( Client client, TransportService transportService, ScriptService scriptService, - ClusterService clusterService + ClusterService clusterService, + TransportSearchAction transportSearchAction ) { super( DeleteByQueryAction.NAME, @@ -71,6 +78,7 @@ public TransportDeleteByQueryAction( this.client = client; this.scriptService = scriptService; this.clusterService = clusterService; + this.transportSearchAction = transportSearchAction; } @Override @@ -95,4 +103,9 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener { +public class TransportUpdateByQueryAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final ThreadPool threadPool; private final Client client; private final ScriptService scriptService; private final ClusterService clusterService; + private final TransportSearchAction transportSearchAction; @Inject public TransportUpdateByQueryAction( @@ -69,7 +75,8 @@ public TransportUpdateByQueryAction( Client client, TransportService transportService, ScriptService scriptService, - ClusterService clusterService + ClusterService clusterService, + TransportSearchAction transportSearchAction ) { super( UpdateByQueryAction.NAME, @@ -81,6 +88,7 @@ public TransportUpdateByQueryAction( this.client = client; this.scriptService = scriptService; this.clusterService = clusterService; + this.transportSearchAction = transportSearchAction; } @Override @@ -107,6 +115,11 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener ); } + @Override + public ResolvedIndices resolveIndices(UpdateByQueryRequest request) { + return transportSearchAction.resolveIndices(request.getSearchRequest()); + } + /** * Simple implementation of update-by-query using scrolling and bulk. */ diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/UpdateByQueryWithScriptTests.java index ce982dcb6bd34..56e3e9cbe311a 100644 --- a/modules/reindex/src/test/java/org/opensearch/index/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/UpdateByQueryWithScriptTests.java @@ -79,6 +79,7 @@ protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptSer null, transportService, scriptService, + null, null ); return new TransportUpdateByQueryAction.AsyncIndexBySearchAction( diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index 11323499efd8b..3924c95a0f58b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -33,11 +33,13 @@ package org.opensearch.action.admin.cluster.shards; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.GroupShardsIterator; import org.opensearch.cluster.routing.ShardIterator; @@ -65,7 +67,7 @@ */ public class TransportClusterSearchShardsAction extends TransportClusterManagerNodeReadAction< ClusterSearchShardsRequest, - ClusterSearchShardsResponse> { + ClusterSearchShardsResponse> implements TransportIndicesResolvingAction { private final IndicesService indicesService; @@ -100,7 +102,7 @@ protected String executor() { @Override protected ClusterBlockException checkBlock(ClusterSearchShardsRequest request, ClusterState state) { return state.blocks() - .indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request)); + .indicesBlockedException(ClusterBlockLevel.METADATA_READ, resolveIndices(state, request).namesOfConcreteIndicesAsArray()); } @Override @@ -115,7 +117,7 @@ protected void clusterManagerOperation( final ActionListener listener ) { ClusterState clusterState = clusterService.state(); - String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request); + String[] concreteIndices = resolveIndices(clusterState, request).namesOfConcreteIndicesAsArray(); Map> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices()); Map indicesAndFilters = new HashMap<>(); Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); @@ -155,4 +157,13 @@ protected void clusterManagerOperation( } listener.onResponse(new ClusterSearchShardsResponse(groupResponses, nodes, indicesAndFilters)); } + + @Override + public ResolvedIndices resolveIndices(ClusterSearchShardsRequest request) { + return ResolvedIndices.of(resolveIndices(clusterService.state(), request)); + } + + private ResolvedIndices.Local.Concrete resolveIndices(ClusterState clusterState, ClusterSearchShardsRequest request) { + return this.indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 0ab19d8b1c63b..e885ac69a28ec 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.RequestValidators; +import org.opensearch.action.admin.indices.delete.DeleteIndexAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; @@ -169,6 +170,7 @@ public void onFailure(Exception t) { public ResolvedIndices resolveIndices(IndicesAliasesRequest request) { try { Set indices = new HashSet<>(); + Set indicesToBeDeleted = new HashSet<>(); for (AliasAction aliasAction : resolvedAliasActions(request, clusterService.state(), false)) { if (aliasAction instanceof AliasAction.Add addAliasAction) { @@ -178,12 +180,15 @@ public ResolvedIndices resolveIndices(IndicesAliasesRequest request) { indices.add(removeAliasAction.getIndex()); indices.add(removeAliasAction.getAlias()); } else if (aliasAction instanceof AliasAction.RemoveIndex removeIndexAction) { - // TODO special action - indices.add(removeIndexAction.getIndex()); + indicesToBeDeleted.add(removeIndexAction.getIndex()); } } - return ResolvedIndices.of(indices); + ResolvedIndices result = ResolvedIndices.of(indices); + if (!indicesToBeDeleted.isEmpty()) { + result = result.withLocalSubActions(DeleteIndexAction.INSTANCE, ResolvedIndices.Local.of(indicesToBeDeleted)); + } + return result; } catch (RuntimeException e) { throw e; } catch (Exception e) { @@ -192,18 +197,24 @@ public ResolvedIndices resolveIndices(IndicesAliasesRequest request) { } } + /** + * Resolves the actions from the IndicesAliasesRequest into concrete AliasAction instances. + * This method has two modes: validate=true makes validation of the parameters and can potentially cause + * exceptions to be thrown upon validation errors. validate=false skips any code that could throw exceptions. This + * is meant for the resolveIndices() method. + */ private List resolvedAliasActions(IndicesAliasesRequest request, ClusterState state, boolean validate) throws Exception { List result = new ArrayList<>(); // Resolve all the AliasActions into AliasAction instances and gather all the aliases for (IndicesAliasesRequest.AliasActions action : request.aliasActions()) { - final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices( + ResolvedIndices.Local.Concrete concreteIndices = indexNameExpressionResolver.concreteResolvedIndices( state, request.indicesOptions(), false, action.indices() ); if (validate) { - for (Index concreteIndex : concreteIndices) { + for (Index concreteIndex : concreteIndices.concreteIndices()) { IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(concreteIndex.getName()); assert indexAbstraction != null : "invalid cluster metadata. index [" + concreteIndex.getName() + "] was not found"; if (indexAbstraction.getParentDataStream() != null) { @@ -216,19 +227,23 @@ private List resolvedAliasActions(IndicesAliasesRequest request, Cl ); } } - final Optional maybeException = requestValidators.validateRequest(request, state, concreteIndices); + final Optional maybeException = requestValidators.validateRequest( + request, + state, + concreteIndices.concreteIndicesAsArray() + ); if (maybeException.isPresent()) { throw maybeException.get(); } } - for (final Index index : concreteIndices) { + for (String index : concreteIndices.namesOfIndices(state)) { switch (action.actionType()) { case ADD: - for (String alias : concreteAliases(action, state.metadata(), index.getName())) { + for (String alias : concreteAliases(action, state.metadata(), index)) { result.add( new AliasAction.Add( - index.getName(), + index, alias, action.filter(), action.indexRouting(), @@ -240,12 +255,12 @@ private List resolvedAliasActions(IndicesAliasesRequest request, Cl } break; case REMOVE: - for (String alias : concreteAliases(action, state.metadata(), index.getName())) { - result.add(new AliasAction.Remove(index.getName(), alias, action.mustExist())); + for (String alias : concreteAliases(action, state.metadata(), index)) { + result.add(new AliasAction.Remove(index, alias, action.mustExist())); } break; case REMOVE_INDEX: - result.add(new AliasAction.RemoveIndex(index.getName())); + result.add(new AliasAction.RemoveIndex(index)); break; default: throw new IllegalArgumentException("Unsupported action [" + action.actionType() + "]"); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 4f4e3bd481ee7..c57cf75cbc263 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -32,6 +32,7 @@ package org.opensearch.action.admin.indices.alias.get; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -39,6 +40,7 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.logging.DeprecationLogger; @@ -65,7 +67,9 @@ * * @opensearch.internal */ -public class TransportGetAliasesAction extends TransportClusterManagerNodeReadAction { +public class TransportGetAliasesAction extends TransportClusterManagerNodeReadAction + implements + TransportIndicesResolvingAction { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TransportGetAliasesAction.class); private final SystemIndices systemIndices; @@ -195,4 +199,31 @@ private static void checkSystemAliasAccess(GetAliasesRequest request, SystemIndi ); } } + + @Override + public ResolvedIndices resolveIndices(GetAliasesRequest request) { + ClusterState state = this.clusterService.state(); + + // The index resolution object in this method is advanced, even though it might not look like it in + // the clusterManagerOperation() method on the first glance. + // + // GetAliasesRequest can be in several different states: + // - no aliases and no indices specified: both the aliases and the indices attribute in GetAliasesRequest are + // empty arrays. This will then cause all aliases and all indices to be resolved and referenced. + // - an alias and no indices: the indices attribute in GetAliasesRequest will be an empty array, which will be + // resolved by indexNameExpressionResolver.concreteIndexNames() to all indices. The action will then filter + // all indices to those that are member of the specified alias + // - no aliases and one or more indices: the aliases attribute in GetAliasesRequest will be an empty array, + // which will be resolved by state.metadata().findAliases() to all aliases, but limited to the aliases + // containing one of the specified indices + // - both aliases and indices specified: this is then the intersection + + String[] concreteIndices; + + try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().newStoredContext(false)) { + concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request); + } + + return ResolvedIndices.of(state.metadata().findAliases(request, concreteIndices).keySet()); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java index 4a0822a9bb754..61f510041b6f3 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -37,12 +37,14 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DestructiveOperations; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataIndexStateService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.ClusterSettings; @@ -67,7 +69,9 @@ * * @opensearch.internal */ -public class TransportCloseIndexAction extends TransportClusterManagerNodeAction { +public class TransportCloseIndexAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportCloseIndexAction.class); @@ -159,7 +163,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) throws Exception { - final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + final Index[] concreteIndices = resolveIndices(state, request).concreteIndicesAsArray(); if (concreteIndices == null || concreteIndices.length == 0) { listener.onResponse(new CloseIndexResponse(true, false, Collections.emptyList())); return; @@ -177,6 +181,15 @@ protected void clusterManagerOperation( })); } + private ResolvedIndices.Local.Concrete resolveIndices(ClusterState state, CloseIndexRequest request) { + return indexNameExpressionResolver.concreteResolvedIndices(state, request); + } + + @Override + public ResolvedIndices resolveIndices(CloseIndexRequest request) { + return ResolvedIndices.of(resolveIndices(clusterService.state(), request)); + } + /** * Reject close index request if cluster mode is [MIXED] and migration direction is [RemoteStore] * @throws IllegalStateException if cluster mode is [MIXED] and migration direction is [RemoteStore] diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java index 9db621b1a5367..91dc4482d12d5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java @@ -38,6 +38,7 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; @@ -49,6 +50,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.MetadataDeleteIndexService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; @@ -169,7 +171,9 @@ public IndicesRequest indices(String... indices) { * * @opensearch.internal */ - public static class TransportAction extends TransportClusterManagerNodeAction { + public static class TransportAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private final MetadataDeleteIndexService deleteIndexService; private final ClusterManagerTaskThrottler.ThrottlingKey removeDataStreamTaskKey; @@ -235,17 +239,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS } static ClusterState removeDataStream(MetadataDeleteIndexService deleteIndexService, ClusterState currentState, Request request) { - Set dataStreams = new HashSet<>(); - Set snapshottingDataStreams = new HashSet<>(); - for (String name : request.names) { - for (String dataStreamName : currentState.metadata().dataStreams().keySet()) { - if (Regex.simpleMatch(name, dataStreamName)) { - dataStreams.add(dataStreamName); - } - } - - snapshottingDataStreams.addAll(SnapshotsService.snapshottingDataStreams(currentState, dataStreams)); - } + Set dataStreams = resolveDataStreams(currentState, request); + Set snapshottingDataStreams = new HashSet<>(SnapshotsService.snapshottingDataStreams(currentState, dataStreams)); if (snapshottingDataStreams.isEmpty() == false) { throw new SnapshotInProgressException( @@ -279,6 +274,23 @@ static ClusterState removeDataStream(MetadataDeleteIndexService deleteIndexServi protected ClusterBlockException checkBlock(Request request, ClusterState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } + + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(resolveDataStreams(clusterService.state(), request)); + } + + private static Set resolveDataStreams(ClusterState state, Request request) { + Set dataStreams = new HashSet<>(); + for (String name : request.names) { + for (String dataStreamName : state.metadata().dataStreams().keySet()) { + if (Regex.simpleMatch(name, dataStreamName)) { + dataStreams.add(dataStreamName); + } + } + } + return dataStreams; + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java index 3641c6b06f34f..e42e9ed433384 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -122,7 +122,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) { - Index[] concreteIndices = resolveIndicesAsArray(request, state); + Index[] concreteIndices = resolveIndices(request, state).concreteIndicesAsArray(); if (concreteIndices.length == 0) { listener.onResponse(new AcknowledgedResponse(true)); return; @@ -149,10 +149,10 @@ public void onFailure(Exception t) { @Override public ResolvedIndices resolveIndices(DeleteIndexRequest request) { - return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + return ResolvedIndices.of(resolveIndices(request, clusterService.state())); } - private Index[] resolveIndicesAsArray(DeleteIndexRequest request, ClusterState clusterState) { - return indexNameExpressionResolver.concreteIndices(clusterState, request); + private ResolvedIndices.Local.Concrete resolveIndices(DeleteIndexRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java index 7dc26641780f3..3a8cfe7c254a4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java @@ -126,7 +126,6 @@ protected void clusterManagerOperation( @Override public ResolvedIndices resolveIndices(IndicesExistsRequest request) { - // TODO this is likely not correct - return ResolvedIndices.of(indexNameExpressionResolver.resolveExpressions(clusterService.state(), request.indices())); + return ResolvedIndices.of(indexNameExpressionResolver.concreteResolvedIndices(clusterService.state(), request)); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java index 53dbb86233803..c074e368bc131 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java @@ -34,8 +34,10 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -55,7 +57,9 @@ * * @opensearch.internal */ -public class TransportGetFieldMappingsAction extends HandledTransportAction { +public class TransportGetFieldMappingsAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final ClusterService clusterService; private final TransportGetFieldMappingsIndexAction shardAction; @@ -78,7 +82,7 @@ public TransportGetFieldMappingsAction( @Override protected void doExecute(Task task, GetFieldMappingsRequest request, final ActionListener listener) { ClusterState clusterState = clusterService.state(); - String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request); + String[] concreteIndices = resolveIndices(request, clusterState).namesOfConcreteIndicesAsArray(); final AtomicInteger indexCounter = new AtomicInteger(); final AtomicInteger completionCounter = new AtomicInteger(concreteIndices.length); final AtomicReferenceArray indexResponses = new AtomicReferenceArray<>(concreteIndices.length); @@ -110,6 +114,15 @@ public void onFailure(Exception e) { } } + private ResolvedIndices.Local.Concrete resolveIndices(GetFieldMappingsRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); + } + + @Override + public ResolvedIndices resolveIndices(GetFieldMappingsRequest request) { + return ResolvedIndices.of(resolveIndices(request, clusterService.state())); + } + private GetFieldMappingsResponse merge(AtomicReferenceArray indexResponses) { Map> mergedResponses = new HashMap<>(); for (int i = 0; i < indexResponses.length(); i++) { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index a156aeeca8e1f..5bb519765fc91 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -134,7 +134,7 @@ protected void clusterManagerOperation( final ActionListener listener ) { try { - final Index[] concreteIndices = resolveIndices(state, request, indexNameExpressionResolver); + final Index[] concreteIndices = resolveIndices(state, request, indexNameExpressionResolver).concreteIndicesAsArray(); final Optional maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices); if (maybeValidationException.isPresent()) { @@ -159,7 +159,11 @@ public ResolvedIndices resolveIndices(PutMappingRequest request) { return ResolvedIndices.of(resolveIndices(clusterService.state(), request, indexNameExpressionResolver)); } - static Index[] resolveIndices(final ClusterState state, PutMappingRequest request, final IndexNameExpressionResolver iner) { + static ResolvedIndices.Local.Concrete resolveIndices( + final ClusterState state, + PutMappingRequest request, + final IndexNameExpressionResolver iner + ) { if (request.getConcreteIndex() == null) { if (request.writeIndexOnly()) { List indices = new ArrayList<>(); @@ -174,12 +178,12 @@ static Index[] resolveIndices(final ClusterState state, PutMappingRequest reques ) ); } - return indices.toArray(Index.EMPTY_ARRAY); + return ResolvedIndices.Local.Concrete.of(indices.toArray(Index.EMPTY_ARRAY)); } else { - return iner.concreteIndices(state, request); + return iner.concreteResolvedIndices(state, request); } } else { - return new Index[] { request.getConcreteIndex() }; + return ResolvedIndices.Local.Concrete.of(request.getConcreteIndex()); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java index a8ecc62fa112c..a1faa865598fb 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -123,7 +123,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) { - final Index[] concreteIndices = resolveIndicesAsArray(request, state); + final Index[] concreteIndices = resolveIndices(request, state).concreteIndicesAsArray(); if (concreteIndices == null || concreteIndices.length == 0) { listener.onResponse(new OpenIndexResponse(true, true)); return; @@ -150,10 +150,10 @@ public void onFailure(Exception t) { @Override public ResolvedIndices resolveIndices(OpenIndexRequest request) { - return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + return ResolvedIndices.of(resolveIndices(request, clusterService.state())); } - private Index[] resolveIndicesAsArray(OpenIndexRequest request, ClusterState clusterState) { - return indexNameExpressionResolver.concreteIndices(clusterState, request); + private ResolvedIndices.Local.Concrete resolveIndices(OpenIndexRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java index 33efa8e691794..75fe3ba63e859 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java @@ -37,12 +37,14 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DestructiveOperations; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataIndexStateService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -65,7 +67,9 @@ * * @opensearch.internal */ -public class TransportAddIndexBlockAction extends TransportClusterManagerNodeAction { +public class TransportAddIndexBlockAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportAddIndexBlockAction.class); @@ -135,7 +139,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) throws Exception { - final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + final Index[] concreteIndices = resolveIndices(state, request).concreteIndicesAsArray(); if (concreteIndices == null || concreteIndices.length == 0) { listener.onResponse(new AddIndexBlockResponse(true, false, Collections.emptyList())); return; @@ -150,4 +154,13 @@ protected void clusterManagerOperation( delegatedListener.onFailure(t); })); } + + private ResolvedIndices.Local.Concrete resolveIndices(ClusterState state, AddIndexBlockRequest request) { + return indexNameExpressionResolver.concreteResolvedIndices(state, request); + } + + @Override + public ResolvedIndices resolveIndices(AddIndexBlockRequest request) { + return ResolvedIndices.of(resolveIndices(this.clusterService.state(), request)); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java index 293df3d3993cd..ac51d845e9e55 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsAction.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.OptionallyResolvedIndices; import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.routing.AllocationId; import org.opensearch.cluster.routing.PlainShardsIterator; @@ -108,9 +109,9 @@ protected void doExecute(Task task, PitSegmentsRequest request, ActionListener listener) { - Index[] concreteIndices = resolveIndicesAsArray(request, state); + Index[] concreteIndices = resolveIndices(request, state).concreteIndicesAsArray(); final Map indexToSettingsBuilder = new HashMap<>(); final Map indexToDefaultSettingsBuilder = new HashMap<>(); for (Index concreteIndex : concreteIndices) { @@ -148,10 +148,10 @@ protected void clusterManagerOperation(GetSettingsRequest request, ClusterState @Override public ResolvedIndices resolveIndices(GetSettingsRequest request) { - return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + return ResolvedIndices.of(resolveIndices(request, clusterService.state())); } - private Index[] resolveIndicesAsArray(GetSettingsRequest request, ClusterState clusterState) { - return indexNameExpressionResolver.concreteIndices(clusterState, request); + private ResolvedIndices.Local.Concrete resolveIndices(GetSettingsRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index 91670ac1d15c9..06a7a8eb8905f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -162,7 +162,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) { - final Index[] concreteIndices = resolveIndicesAsArray(request, state); + final Index[] concreteIndices = resolveIndices(request, state).concreteIndicesAsArray(); UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest().indices( concreteIndices ) @@ -187,10 +187,10 @@ public void onFailure(Exception t) { @Override public ResolvedIndices resolveIndices(UpdateSettingsRequest request) { - return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + return ResolvedIndices.of(resolveIndices(request, clusterService.state())); } - private Index[] resolveIndicesAsArray(UpdateSettingsRequest request, ClusterState clusterState) { - return indexNameExpressionResolver.concreteIndices(clusterState, request); + private ResolvedIndices.Local.Concrete resolveIndices(UpdateSettingsRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index fad504a476511..a455113d6c849 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -36,6 +36,7 @@ import org.apache.lucene.util.CollectionUtil; import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.ClusterState; @@ -45,6 +46,7 @@ import org.opensearch.cluster.health.ClusterShardHealth; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; @@ -84,7 +86,7 @@ */ public class TransportIndicesShardStoresAction extends TransportClusterManagerNodeReadAction< IndicesShardStoresRequest, - IndicesShardStoresResponse> { + IndicesShardStoresResponse> implements TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportIndicesShardStoresAction.class); @@ -133,7 +135,7 @@ protected void clusterManagerOperation( ) { final RoutingTable routingTables = state.routingTable(); final RoutingNodes routingNodes = state.getRoutingNodes(); - final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request); + final String[] concreteIndices = resolveIndices(state, request).namesOfConcreteIndicesAsArray(); final Set> shardsToFetch = new HashSet<>(); logger.trace("using cluster state version [{}] to determine shards", state.version()); @@ -161,10 +163,19 @@ protected void clusterManagerOperation( new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardsToFetch, listener, clusterManagerMetrics).start(); } + private ResolvedIndices.Local.Concrete resolveIndices(ClusterState state, IndicesShardStoresRequest request) { + return indexNameExpressionResolver.concreteResolvedIndices(state, request); + } + + @Override + public ResolvedIndices resolveIndices(IndicesShardStoresRequest request) { + return ResolvedIndices.of(resolveIndices(clusterService.state(), request)); + } + @Override protected ClusterBlockException checkBlock(IndicesShardStoresRequest request, ClusterState state) { return state.blocks() - .indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request)); + .indicesBlockedException(ClusterBlockLevel.METADATA_READ, resolveIndices(state, request).namesOfConcreteIndicesAsArray()); } /** diff --git a/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/pause/TransportPauseIngestionAction.java b/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/pause/TransportPauseIngestionAction.java index 6f2a2577c84a3..92d8fc10efc56 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/pause/TransportPauseIngestionAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/pause/TransportPauseIngestionAction.java @@ -27,13 +27,11 @@ import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.index.Index; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.io.IOException; -import java.util.Arrays; /** * Pause ingestion transport action. @@ -110,20 +108,22 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) throws Exception { - final Index[] concreteIndices = resolveIndicesAsArray(request, state); - if (concreteIndices == null || concreteIndices.length == 0) { + final ResolvedIndices.Local.Concrete concreteIndices = resolveIndices(request, state); + if (concreteIndices.concreteIndices().isEmpty()) { listener.onResponse(new PauseIngestionResponse(true, false, new IngestionStateShardFailure[0], "")); return; } - String[] indices = Arrays.stream(concreteIndices).map(Index::getName).toArray(String[]::new); - UpdateIngestionStateRequest updateIngestionStateRequest = new UpdateIngestionStateRequest(indices, new int[0]); + UpdateIngestionStateRequest updateIngestionStateRequest = new UpdateIngestionStateRequest( + concreteIndices.namesOfConcreteIndicesAsArray(), + new int[0] + ); updateIngestionStateRequest.timeout(request.clusterManagerNodeTimeout()); updateIngestionStateRequest.setIngestionPaused(true); ingestionStateService.updateIngestionPollerState( "pause-ingestion", - concreteIndices, + concreteIndices.concreteIndicesAsArray(), updateIngestionStateRequest, new ActionListener<>() { @@ -151,10 +151,10 @@ public void onFailure(Exception e) { @Override public ResolvedIndices resolveIndices(PauseIngestionRequest request) { - return ResolvedIndices.of(resolveIndicesAsArray(request, clusterService.state())); + return ResolvedIndices.of(resolveIndices(request, clusterService.state())); } - private Index[] resolveIndicesAsArray(PauseIngestionRequest request, ClusterState clusterState) { - return indexNameExpressionResolver.concreteIndices(clusterState, request); + private ResolvedIndices.Local.Concrete resolveIndices(PauseIngestionRequest request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/resume/TransportResumeIngestionAction.java b/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/resume/TransportResumeIngestionAction.java index dea46c7cf6b23..02b58df74dd86 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/resume/TransportResumeIngestionAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/streamingingestion/resume/TransportResumeIngestionAction.java @@ -15,12 +15,14 @@ import org.opensearch.action.admin.indices.streamingingestion.state.UpdateIngestionStateResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DestructiveOperations; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataStreamingIngestionStateService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -38,7 +40,9 @@ * * @opensearch.experimental */ -public class TransportResumeIngestionAction extends TransportClusterManagerNodeAction { +public class TransportResumeIngestionAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportResumeIngestionAction.class); @@ -106,7 +110,7 @@ protected void clusterManagerOperation( final ClusterState state, final ActionListener listener ) throws Exception { - final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + final Index[] concreteIndices = resolveIndices(state, request).concreteIndicesAsArray(); if (concreteIndices == null || concreteIndices.length == 0) { listener.onResponse(new ResumeIngestionResponse(true, false, new IngestionStateShardFailure[0], "")); return; @@ -173,4 +177,13 @@ private UpdateIngestionStateRequest getIngestionResumeRequest(String[] indices, return updateIngestionStateRequest; } + + private ResolvedIndices.Local.Concrete resolveIndices(ClusterState state, ResumeIngestionRequest request) { + return indexNameExpressionResolver.concreteResolvedIndices(state, request); + } + + @Override + public ResolvedIndices resolveIndices(ResumeIngestionRequest request) { + return ResolvedIndices.of(resolveIndices(clusterService.state(), request)); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/tiering/TransportHotToWarmTieringAction.java b/server/src/main/java/org/opensearch/action/admin/indices/tiering/TransportHotToWarmTieringAction.java index 8d1ab0bb37cdd..fce2146054767 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/tiering/TransportHotToWarmTieringAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/tiering/TransportHotToWarmTieringAction.java @@ -11,12 +11,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterInfoService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.routing.allocation.DiskThresholdSettings; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; @@ -39,7 +41,9 @@ * @opensearch.experimental */ @ExperimentalApi -public class TransportHotToWarmTieringAction extends TransportClusterManagerNodeAction { +public class TransportHotToWarmTieringAction extends TransportClusterManagerNodeAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportHotToWarmTieringAction.class); private final ClusterInfoService clusterInfoService; @@ -90,7 +94,7 @@ protected void clusterManagerOperation( ClusterState state, ActionListener listener ) throws Exception { - Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); + Index[] concreteIndices = resolveIndices(state, request).concreteIndicesAsArray(); if (concreteIndices == null || concreteIndices.length == 0) { listener.onResponse(new HotToWarmTieringResponse(true)); return; @@ -107,4 +111,13 @@ protected void clusterManagerOperation( return; } } + + private ResolvedIndices.Local.Concrete resolveIndices(ClusterState state, TieringIndexRequest request) { + return indexNameExpressionResolver.concreteResolvedIndices(state, request); + } + + @Override + public ResolvedIndices resolveIndices(TieringIndexRequest request) { + return ResolvedIndices.of(resolveIndices(clusterService.state(), request)); + } } diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java index 7490eb9f6afa7..e9e9cb7f37532 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java @@ -224,7 +224,7 @@ protected void handlePrimaryTermValidationRequest( @Override public ResolvedIndices resolveIndices(BulkShardRequest request) { - return ResolvedIndices.of(request.shardId().getIndexName()); + return ResolvedIndices.of(request.index()); } /** diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index bef037a479c0b..f9b9691d8b307 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -43,7 +43,6 @@ import org.opensearch.common.inject.Inject; import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.Strings; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.RemoteClusterAware; @@ -177,12 +176,12 @@ private ResolvedIndices resolveIndices(FieldCapabilitiesRequest request, Cluster idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState) ); final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); - final String[] concreteIndices; + final ResolvedIndices.Local.Concrete concreteIndices; if (localIndices == null) { // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices - concreteIndices = Strings.EMPTY_ARRAY; + concreteIndices = ResolvedIndices.Local.Concrete.empty(); } else { - concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, localIndices); + concreteIndices = indexNameExpressionResolver.concreteResolvedIndices(clusterState, localIndices); } return ResolvedIndices.of(concreteIndices).withLocalOriginalIndices(localIndices).withRemoteIndices(remoteClusterIndices); diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 52937182e6a63..73cb878d192f8 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -41,10 +41,12 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.ChannelActionListener; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.FailAwareWeightedRouting; @@ -93,7 +95,7 @@ */ public class TransportFieldCapabilitiesIndexAction extends HandledTransportAction< FieldCapabilitiesIndexRequest, - FieldCapabilitiesIndexResponse> { + FieldCapabilitiesIndexResponse> implements TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportFieldCapabilitiesIndexAction.class); @@ -213,6 +215,11 @@ private ClusterBlockException checkRequestBlock(ClusterState state, String concr return state.blocks().indexBlockedException(ClusterBlockLevel.READ, concreteIndex); } + @Override + public ResolvedIndices resolveIndices(FieldCapabilitiesIndexRequest request) { + return ResolvedIndices.of(request.index()); + } + /** * An action that executes on each shard sequentially until it finds one that can match the provided * {@link FieldCapabilitiesIndexRequest#indexFilter()}. In which case the shard is used diff --git a/server/src/main/java/org/opensearch/action/search/CreatePitController.java b/server/src/main/java/org/opensearch/action/search/CreatePitController.java index 87eb27bdb8255..1b50fb48d5d7e 100644 --- a/server/src/main/java/org/opensearch/action/search/CreatePitController.java +++ b/server/src/main/java/org/opensearch/action/search/CreatePitController.java @@ -15,6 +15,7 @@ import org.opensearch.action.StepListener; import org.opensearch.action.support.GroupedActionListener; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; @@ -86,11 +87,7 @@ public void executeCreatePit( StepListener createPitListener, ActionListener updatePitIdListener ) { - SearchRequest searchRequest = new SearchRequest(request.getIndices()); - searchRequest.preference(request.getPreference()); - searchRequest.routing(request.getRouting()); - searchRequest.indicesOptions(request.getIndicesOptions()); - searchRequest.allowPartialSearchResults(request.shouldAllowPartialPitCreation()); + SearchRequest searchRequest = request.toSearchRequest(); SearchTask searchTask = searchRequest.createTask( task.getId(), task.getType(), @@ -326,4 +323,8 @@ public void onFailure(Exception e) { } pitService.deletePitContexts(nodeToContextsMap, deleteListener); } + + ResolvedIndices resolveIndices(CreatePitRequest request) { + return transportSearchAction.resolveIndices(request.toSearchRequest()); + } } diff --git a/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java b/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java index 840d4becda714..f6791da13a328 100644 --- a/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java +++ b/server/src/main/java/org/opensearch/action/search/CreatePitRequest.java @@ -196,4 +196,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } return builder; } + + SearchRequest toSearchRequest() { + SearchRequest searchRequest = new SearchRequest(this.getIndices()); + searchRequest.preference(this.getPreference()); + searchRequest.routing(this.getRouting()); + searchRequest.indicesOptions(this.getIndicesOptions()); + searchRequest.allowPartialSearchResults(this.shouldAllowPartialPitCreation()); + return searchRequest; + } } diff --git a/server/src/main/java/org/opensearch/action/search/TransportCreatePitAction.java b/server/src/main/java/org/opensearch/action/search/TransportCreatePitAction.java index baa113997f243..e15d216bdabe0 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportCreatePitAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportCreatePitAction.java @@ -12,6 +12,8 @@ import org.opensearch.action.StepListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.unit.TimeValue; @@ -32,7 +34,9 @@ /** * Transport action for creating PIT reader context */ -public class TransportCreatePitAction extends HandledTransportAction { +public class TransportCreatePitAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { public static final String CREATE_PIT_ACTION = "create_pit"; private final TransportService transportService; @@ -76,6 +80,11 @@ protected void doExecute(Task task, CreatePitRequest request, ActionListener buildPerIndexAliasFilter( SearchRequest request, ClusterState clusterState, - Index[] concreteIndices, + ResolvedIndices.Local.Concrete concreteIndices, Map remoteAliasMap ) { final Map aliasFilterMap = new HashMap<>(); final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); - for (Index index : concreteIndices) { + for (Index index : concreteIndices.concreteIndices()) { clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index.getName()); AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, index.getName(), indicesAndAliases); assert aliasFilter != null; @@ -994,7 +994,7 @@ static List getRemoteShardsIteratorFromPointInTime( public ResolvedIndices resolveIndices(SearchRequest searchRequest) { ClusterState clusterState = clusterService.state(); OriginalIndicesAndSearchContextId requestedIndices = extractRequestedIndices(searchRequest, clusterState); - Index[] localConcreteIndices = resolveLocalIndices( + ResolvedIndices.Local.Concrete localConcreteIndices = resolveLocalIndices( requestedIndices.localOriginalIndices, clusterState, new SearchTimeProvider(searchRequest.getOrCreateAbsoluteStartMillis(), System.nanoTime(), System::nanoTime) @@ -1021,11 +1021,16 @@ private OriginalIndicesAndSearchContextId extractRequestedIndices(SearchRequest return new OriginalIndicesAndSearchContextId(localIndices, remoteClusterIndices, searchContext); } - private Index[] resolveLocalIndices(OriginalIndices localIndices, ClusterState clusterState, SearchTimeProvider timeProvider) { + private ResolvedIndices.Local.Concrete resolveLocalIndices( + OriginalIndices localIndices, + ClusterState clusterState, + SearchTimeProvider timeProvider + ) { if (localIndices == null) { - return Index.EMPTY_ARRAY; // don't search on any local index (happens when only remote indices were specified) + // don't search on any local index (happens when only remote indices were specified) + return ResolvedIndices.Local.Concrete.empty(); } - return indexNameExpressionResolver.concreteIndices(clusterState, localIndices, timeProvider.getAbsoluteStartMillis()); + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, localIndices, timeProvider.getAbsoluteStartMillis()); } private void executeSearch( @@ -1065,17 +1070,14 @@ private void executeSearch( searchRequest.pointInTimeBuilder().getKeepAlive() ); } else { - final Index[] indices = resolveLocalIndices(localIndices, clusterState, timeProvider); + ResolvedIndices.Local.Concrete indices = resolveLocalIndices(localIndices, clusterState, timeProvider); Map> routingMap = indexNameExpressionResolver.resolveSearchRouting( clusterState, searchRequest.routing(), searchRequest.indices() ); routingMap = routingMap == null ? Collections.emptyMap() : Collections.unmodifiableMap(routingMap); - concreteLocalIndices = new String[indices.length]; - for (int i = 0; i < indices.length; i++) { - concreteLocalIndices[i] = indices[i].getName(); - } + concreteLocalIndices = indices.namesOfConcreteIndicesAsArray(); Map nodeSearchCounts = searchTransportService.getPendingSearchRequests(); SliceBuilder slice = searchRequest.source() == null ? null : searchRequest.source().slice(); GroupShardsIterator localShardRoutings = clusterService.operationRouting() diff --git a/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java b/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java index 20b183309a09d..f999d34b16124 100644 --- a/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java +++ b/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java @@ -9,11 +9,10 @@ package org.opensearch.action.support; import org.opensearch.action.ActionRequest; +import org.opensearch.cluster.metadata.OptionallyResolvedIndices; import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.core.action.ActionResponse; -import java.util.Optional; - /** * This class can be used to provide metadata about action requests to ActionFilter implementations. * At the moment, this class provides information about the requested indices of a request, but it can be @@ -35,8 +34,7 @@ public static A private final TransportAction transportAction; private final Request request; - private ResolvedIndices resolvedIndices; - private boolean resolvedIndicesInitialized; + private OptionallyResolvedIndices resolvedIndices; ActionRequestMetadata(TransportAction transportAction, Request request) { this.transportAction = transportAction; @@ -48,15 +46,16 @@ public static A * expressions or patterns will be resolved. *

* If the request cannot reference indices OR if the respective action does not support resolving of requests, - * this returns an empty Optional. + * this returns an {@link OptionallyResolvedIndices} with unknown = true. If indices can be resolved, actually + * a {@link ResolvedIndices} object will be returned. */ - public Optional resolvedIndices() { + public OptionallyResolvedIndices resolvedIndices() { if (!(transportAction instanceof TransportIndicesResolvingAction)) { - return Optional.empty(); + return OptionallyResolvedIndices.unknown(); } - if (this.resolvedIndicesInitialized) { - return Optional.of(this.resolvedIndices); + if (this.resolvedIndices != null) { + return this.resolvedIndices; } else { return resolveIndices(); } @@ -66,13 +65,12 @@ public Optional resolvedIndices() { * Performs the actual index resolution. Index resolution can be relatively costly on big clusters, so we * perform it lazily only when requested. */ - private Optional resolveIndices() { + private OptionallyResolvedIndices resolveIndices() { @SuppressWarnings("unchecked") TransportIndicesResolvingAction indicesResolvingAction = (TransportIndicesResolvingAction) this.transportAction; - ResolvedIndices result = indicesResolvingAction.resolveIndices(request); + OptionallyResolvedIndices result = indicesResolvingAction.resolveIndices(request); this.resolvedIndices = result; - this.resolvedIndicesInitialized = true; - return Optional.of(result); + return result; } } diff --git a/server/src/main/java/org/opensearch/action/support/TransportIndicesResolvingAction.java b/server/src/main/java/org/opensearch/action/support/TransportIndicesResolvingAction.java index 120117ade6466..73024a5bbdc0f 100644 --- a/server/src/main/java/org/opensearch/action/support/TransportIndicesResolvingAction.java +++ b/server/src/main/java/org/opensearch/action/support/TransportIndicesResolvingAction.java @@ -9,7 +9,7 @@ package org.opensearch.action.support; import org.opensearch.action.ActionRequest; -import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.metadata.OptionallyResolvedIndices; /** * An additional interface that should be implemented by TransportAction implementations which need to resolve @@ -26,5 +26,5 @@ public interface TransportIndicesResolvingAction /** * Returns the actual indices the action will operate on, given the specified request and cluster state. */ - ResolvedIndices resolveIndices(Request request); + OptionallyResolvedIndices resolveIndices(Request request); } diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java index 06c137d0fb477..fe4449516be86 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java @@ -113,11 +113,11 @@ protected void doExecute(Task task, Request request, ActionListener li @Override public ResolvedIndices resolveIndices(Request request) { - return ResolvedIndices.of(indexNameExpressionResolver.concreteIndexNames(clusterService.state(), request)); + return ResolvedIndices.of(resolveIndices(request, clusterService.state())); } - protected ResolvedIndices resolveIndices(Request request, ClusterState clusterState) { - return ResolvedIndices.of(indexNameExpressionResolver.concreteIndexNames(clusterState, request)); + protected ResolvedIndices.Local.Concrete resolveIndices(Request request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); } protected abstract Response newResponse(Request request, AtomicReferenceArray shardsResponses, ClusterState clusterState); @@ -167,7 +167,7 @@ protected AsyncBroadcastAction(Task task, Request request, ActionListener results, List li } @Override - public ResolvedIndices resolveIndices(Request request) { - return ResolvedIndices.of(resolveConcreteIndexNames(clusterService.state(), request)); + public OptionallyResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(resolveConcreteIndices(clusterService.state(), request)); } /** @@ -318,7 +319,7 @@ protected AsyncAction(Task task, Request request, ActionListener liste throw globalBlockException; } - String[] concreteIndices = resolveConcreteIndexNames(clusterState, request); + String[] concreteIndices = resolveConcreteIndices(clusterState, request).namesOfConcreteIndicesAsArray(); ClusterBlockException requestBlockException = checkRequestBlock(clusterState, request, concreteIndices); if (requestBlockException != null) { throw requestBlockException; diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java index 883d2e7429e2d..fec73fc596ae3 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java @@ -32,11 +32,13 @@ package org.opensearch.action.support.clustermanager.info; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeReadAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; @@ -50,7 +52,9 @@ * @opensearch.internal */ public abstract class TransportClusterInfoAction, Response extends ActionResponse> extends - TransportClusterManagerNodeReadAction { + TransportClusterManagerNodeReadAction + implements + TransportIndicesResolvingAction { public TransportClusterInfoAction( String actionName, @@ -74,15 +78,24 @@ protected String executor() { @Override protected ClusterBlockException checkBlock(Request request, ClusterState state) { return state.blocks() - .indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request)); + .indicesBlockedException(ClusterBlockLevel.METADATA_READ, resolveIndices(state, request).namesOfConcreteIndicesAsArray()); } @Override protected final void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) { - String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request); + String[] concreteIndices = resolveIndices(state, request).namesOfConcreteIndicesAsArray(); doClusterManagerOperation(request, concreteIndices, state, listener); } + @Override + public ResolvedIndices resolveIndices(Request request) { + return ResolvedIndices.of(resolveIndices(clusterService.state(), request)); + } + + private ResolvedIndices.Local.Concrete resolveIndices(ClusterState state, Request request) { + return indexNameExpressionResolver.concreteResolvedIndices(state, request); + } + protected abstract void doClusterManagerOperation( Request request, String[] concreteIndices, diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java index c13cffe598456..a1c0dd99e3c44 100644 --- a/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java +++ b/server/src/main/java/org/opensearch/action/support/replication/TransportBroadcastReplicationAction.java @@ -145,11 +145,11 @@ public void onFailure(Exception e) { @Override public ResolvedIndices resolveIndices(Request request) { - return resolveIndices(request, clusterService.state()); + return ResolvedIndices.of(resolveIndices(request, clusterService.state())); } - private ResolvedIndices resolveIndices(Request request, ClusterState clusterState) { - return ResolvedIndices.of(indexNameExpressionResolver.concreteIndexNames(clusterState, request)); + private ResolvedIndices.Local.Concrete resolveIndices(Request request, ClusterState clusterState) { + return indexNameExpressionResolver.concreteResolvedIndices(clusterState, request); } protected void shardExecute(Task task, Request request, ShardId shardId, ActionListener shardActionListener) { @@ -163,7 +163,7 @@ protected void shardExecute(Task task, Request request, ShardId shardId, ActionL */ protected List shards(Request request, ClusterState clusterState) { List shardIds = new ArrayList<>(); - Set concreteIndices = resolveIndices(request, clusterState).local().names(); + Set concreteIndices = resolveIndices(request, clusterState).namesOfConcreteIndices(); for (String index : concreteIndices) { IndexMetadata indexMetadata = clusterState.metadata().getIndices().get(index); if (indexMetadata != null) { diff --git a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java index f9ef23c711180..0317275bd90d4 100644 --- a/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationAction.java @@ -116,7 +116,6 @@ public ResolvedIndices resolveIndices(Request request) { return ResolvedIndices.of(request.concreteIndex()); } else { try { - // TODO shall we possibly also set request.concreteIndex here? return ResolvedIndices.of(indexNameExpressionResolver.concreteWriteIndex(clusterService.state(), request).getName()); } catch (IndexNotFoundException e) { // We just return the original unresolved expression. The error we encountered here will diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java index abfabdd10c340..d9a1e66919fec 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java @@ -144,7 +144,7 @@ public String[] concreteIndexNamesWithSystemIndexAccess(ClusterState state, Indi * Same as {@link #concreteIndices(ClusterState, IndicesOptions, String...)}, but the index expressions and options * are encapsulated in the specified request and resolves data streams. */ - public Index[] concreteIndices(ClusterState state, IndicesRequest request) { + public ResolvedIndices.Local.Concrete concreteResolvedIndices(ClusterState state, IndicesRequest request) { Context context = new Context( state, request.indicesOptions(), @@ -153,7 +153,15 @@ public Index[] concreteIndices(ClusterState state, IndicesRequest request) { request.includeDataStreams(), isSystemIndexAccessAllowed() ); - return concreteIndices(context, request.indices()); + return concreteResolvedIndices(context, request.indices()); + } + + /** + * Same as {@link #concreteIndices(ClusterState, IndicesOptions, String...)}, but the index expressions and options + * are encapsulated in the specified request and resolves data streams. + */ + public Index[] concreteIndices(ClusterState state, IndicesRequest request) { + return concreteResolvedIndices(state, request).concreteIndicesAsArray(); } /** @@ -207,6 +215,32 @@ public List dataStreamNames(ClusterState state, IndicesOptions options, .collect(Collectors.toList()); } + /** + * Translates the provided index expression into actual concrete indices, properly deduplicated. + * + * @param state the cluster state containing all the data to resolve to expressions to concrete indices + * @param options defines how the aliases or indices need to be resolved to concrete indices + * @param indexExpressions expressions that can be resolved to alias or index names. + * @return the resolved concrete indices based on the cluster state, indices options and index expressions. Any errors + * encountered during the resolution will be not thrown immediately as an exception, but only when you try + * to access the concrete indices from the ResolvedIndices.Local.Concrete object. You can use the method + * names() to access the names of the requested indices in a way that is safe from thrown exceptions due to + * resolution errors. + */ + public ResolvedIndices.Local.Concrete concreteResolvedIndices(ClusterState state, IndicesOptions options, String... indexExpressions) { + return concreteResolvedIndices(state, options, false, indexExpressions); + } + + public ResolvedIndices.Local.Concrete concreteResolvedIndices( + ClusterState state, + IndicesOptions options, + boolean includeDataStreams, + String... indexExpressions + ) { + Context context = new Context(state, options, false, false, includeDataStreams, isSystemIndexAccessAllowed()); + return concreteResolvedIndices(context, indexExpressions); + } + /** * Translates the provided index expression into actual concrete indices, properly deduplicated. * @@ -225,8 +259,7 @@ public Index[] concreteIndices(ClusterState state, IndicesOptions options, Strin } public Index[] concreteIndices(ClusterState state, IndicesOptions options, boolean includeDataStreams, String... indexExpressions) { - Context context = new Context(state, options, false, false, includeDataStreams, isSystemIndexAccessAllowed()); - return concreteIndices(context, indexExpressions); + return concreteResolvedIndices(state, options, includeDataStreams, indexExpressions).concreteIndicesAsArray(); } /** @@ -235,13 +268,13 @@ public Index[] concreteIndices(ClusterState state, IndicesOptions options, boole * @param state the cluster state containing all the data to resolve to expressions to concrete indices * @param startTime The start of the request where concrete indices is being invoked for * @param request request containing expressions that can be resolved to alias, index, or data stream names. - * @return the resolved concrete indices based on the cluster state, indices options and index expressions - * provided indices options in the context don't allow such a case, or if the final result of the indices resolution - * contains no indices and the indices options in the context don't allow such a case. - * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided - * indices options in the context don't allow such a case. + * @return the resolved concrete indices based on the cluster state, indices options and index expressions. Any errors + * encountered during the resolution will be not thrown immediately as an exception, but only when you try + * to access the concrete indices from the ResolvedIndices.Local.Concrete object. You can use the method + * names() to access the names of the requested indices in a way that is safe from thrown exceptions due to + * resolution errors. */ - public Index[] concreteIndices(ClusterState state, IndicesRequest request, long startTime) { + public ResolvedIndices.Local.Concrete concreteResolvedIndices(ClusterState state, IndicesRequest request, long startTime) { Context context = new Context( state, request.indicesOptions(), @@ -252,24 +285,40 @@ public Index[] concreteIndices(ClusterState state, IndicesRequest request, long false, isSystemIndexAccessAllowed() ); - return concreteIndices(context, request.indices()); + return concreteResolvedIndices(context, request.indices()); + } + + /** + * Translates the provided index expression into actual concrete indices, properly deduplicated. + * + * @param state the cluster state containing all the data to resolve to expressions to concrete indices + * @param startTime The start of the request where concrete indices is being invoked for + * @param request request containing expressions that can be resolved to alias, index, or data stream names. + * @return the resolved concrete indices based on the cluster state, indices options and index expressions + * provided indices options in the context don't allow such a case, or if the final result of the indices resolution + * contains no indices and the indices options in the context don't allow such a case. + * @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided + * indices options in the context don't allow such a case. + */ + public Index[] concreteIndices(ClusterState state, IndicesRequest request, long startTime) { + return concreteResolvedIndices(state, request, startTime).concreteIndicesAsArray(); } String[] concreteIndexNames(Context context, String... indexExpressions) { - Index[] indexes = concreteIndices(context, indexExpressions); - String[] names = new String[indexes.length]; - for (int i = 0; i < indexes.length; i++) { - names[i] = indexes[i].getName(); - } - return names; + return concreteResolvedIndices(context, indexExpressions).namesOfConcreteIndicesAsArray(); } Index[] concreteIndices(Context context, String... indexExpressions) { + return concreteResolvedIndices(context, indexExpressions).concreteIndicesAsArray(); + } + + ResolvedIndices.Local.Concrete concreteResolvedIndices(Context context, String... indexExpressions) { if (indexExpressions == null || indexExpressions.length == 0) { indexExpressions = new String[] { Metadata.ALL }; } Metadata metadata = context.getState().metadata(); IndicesOptions options = context.getOptions(); + context.resolutionErrors.clear(); // If only one index is specified then whether we fail a request if an index is missing depends on the allow_no_indices // option. At some point we should change this, because there shouldn't be a reason why whether a single index // or multiple indices are specified yield different behaviour. @@ -292,14 +341,18 @@ Index[] concreteIndices(Context context, String... indexExpressions) { infe = new IndexNotFoundException((String) null); } infe.setResources("index_expression", indexExpressions); - throw infe; + return ResolvedIndices.Local.Concrete.empty() + .withResolutionErrors(context.getResolutionErrors()) + .withResolutionErrors(infe); } else { - return Index.EMPTY_ARRAY; + return ResolvedIndices.Local.Concrete.empty(); } } boolean excludedDataStreams = false; final Set concreteIndices = new HashSet<>(expressions.size()); + Set acceptedExpressions = new HashSet<>(expressions.size()); + for (String expression : expressions) { IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(expression); if (indexAbstraction == null) { @@ -311,34 +364,38 @@ Index[] concreteIndices(Context context, String... indexExpressions) { infe = new IndexNotFoundException(expression); } infe.setResources("index_expression", expression); - throw infe; - } else { - continue; + context.addResolutionError(infe); } + acceptedExpressions.add(expression); + continue; } else if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.getOptions().ignoreAliases()) { if (failNoIndices) { - throw aliasesNotSupportedException(expression); - } else { - continue; + context.addResolutionError(aliasesNotSupportedException(expression)); } + continue; } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.includeDataStreams() == false) { excludedDataStreams = true; continue; } + acceptedExpressions.add(expression); + if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.isResolveToWriteIndex()) { IndexMetadata writeIndex = indexAbstraction.getWriteIndex(); if (writeIndex == null) { - throw new IllegalArgumentException( - "no write index is defined for alias [" - + indexAbstraction.getName() - + "]." - + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" - + " indices without one being designated as a write index" + context.addResolutionError( + new IllegalArgumentException( + "no write index is defined for alias [" + + indexAbstraction.getName() + + "]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index" + ) ); - } - if (addIndex(writeIndex, context)) { - concreteIndices.add(writeIndex.getIndex()); + } else { + if (addIndex(writeIndex, context)) { + concreteIndices.add(writeIndex.getIndex()); + } } } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.isResolveToWriteIndex()) { IndexMetadata writeIndex = indexAbstraction.getWriteIndex(); @@ -352,19 +409,26 @@ Index[] concreteIndices(Context context, String... indexExpressions) { for (IndexMetadata indexMetadata : indexAbstraction.getIndices()) { indexNames[i++] = indexMetadata.getIndex().getName(); } - throw new IllegalArgumentException( - indexAbstraction.getType().getDisplayName() - + " [" - + expression - + "] has more than one index associated with it " - + Arrays.toString(indexNames) - + ", can't execute a single index op" + context.addResolutionError( + new IllegalArgumentException( + indexAbstraction.getType().getDisplayName() + + " [" + + expression + + "] has more than one index associated with it " + + Arrays.toString(indexNames) + + ", can't execute a single index op" + ) ); + continue; } for (IndexMetadata index : indexAbstraction.getIndices()) { - if (shouldTrackConcreteIndex(context, options, index)) { - concreteIndices.add(index.getIndex()); + try { + if (shouldTrackConcreteIndex(context, options, index)) { + concreteIndices.add(index.getIndex()); + } + } catch (IndexClosedException e) { + context.addResolutionError(e); } } } @@ -377,10 +441,10 @@ Index[] concreteIndices(Context context, String... indexExpressions) { // Allows callers to handle IndexNotFoundException differently based on whether data streams were excluded. infe.addMetadata(EXCLUDED_DATA_STREAMS_KEY, "true"); } - throw infe; + context.addResolutionError(infe); } checkSystemIndexAccess(context, metadata, concreteIndices, indexExpressions); - return concreteIndices.toArray(new Index[0]); + return ResolvedIndices.Local.Concrete.of(concreteIndices, acceptedExpressions).withResolutionErrors(context.getResolutionErrors()); } private void checkSystemIndexAccess(Context context, Metadata metadata, Set concreteIndices, String[] originalPatterns) { @@ -816,6 +880,7 @@ public static class Context { private final boolean includeDataStreams; private final boolean preserveDataStreams; private final boolean isSystemIndexAccessAllowed; + private final List resolutionErrors = new ArrayList<>(); public Context(ClusterState state, IndicesOptions options, boolean isSystemIndexAccessAllowed) { this(state, options, System.currentTimeMillis(), isSystemIndexAccessAllowed); @@ -929,6 +994,28 @@ public boolean isPreserveDataStreams() { public boolean isSystemIndexAccessAllowed() { return isSystemIndexAccessAllowed; } + + /** + * Adds a new error encountered while resolving index resolution. If this method is used, index resolution + * will continue until it is completed. However, after completion The methods concreteIndices(), + * concreteIndexNames(), concreteWriteIndex(), etc. will howe will throw the first resolution error that was + * tracked with this method. The method concreteResolvedIndices() can be used to retrieve the resolution + * result without a thrown exception. + * + * @param resolutionError the encountered resolution error. Ideally this extends OpenSearchException. + */ + public void addResolutionError(RuntimeException resolutionError) { + this.resolutionErrors.add(resolutionError); + } + + /** + * Returns the currently encountered resolution errors. + * Note: This is on purpose package private. External resolvers should have no need to inspect the + * present resolution errors. + */ + private List getResolutionErrors() { + return this.resolutionErrors; + } } /** @@ -996,11 +1083,6 @@ public List resolve(Context context, List expressions) { if (result == null) { return expressions; } - if (result.isEmpty() && !options.allowNoIndices()) { - IndexNotFoundException infe = new IndexNotFoundException((String) null); - infe.setResources("index_or_alias", expressions.toArray(new String[0])); - throw infe; - } return new ArrayList<>(result); } @@ -1010,9 +1092,9 @@ private Set innerResolve(Context context, List expressions, Indi for (int i = 0; i < expressions.size(); i++) { String expression = expressions.get(i); if (Strings.isEmpty(expression)) { - throw indexNotFoundException(expression); + context.addResolutionError(indexNotFoundException(expression)); } - validateAliasOrIndex(expression); + validateAliasOrIndex(context, expression); if (aliasOrIndexExists(context, options, metadata, expression)) { if (result != null) { result.add(expression); @@ -1031,16 +1113,26 @@ private Set innerResolve(Context context, List expressions, Indi result = new HashSet<>(expressions.subList(0, i)); } if (Regex.isSimpleMatchPattern(expression) == false) { - // TODO why does wildcard resolver throw exceptions regarding non wildcarded expressions? This should not be done here. + // The following code enforces the ignoreUnavailable rule for non-wildcard expressions. + // This might seem to be non-intuitive, as we are in the class WildcardExpressionResolver which + // should be responsible only for wildcard expressions. However - given the current architecture - + // there is no good other place to perform this logic: + // - We cannot perform it before the resolver chain, as we still might have expressions which we + // just cannot understand. So, we have to wait for all other resolvers to complete. + // - We cannot perform it efficiently after the resolver chain, as then the WildcardExpressionResolver + // will have expanded wildcards to potentially many indices. If we did these checks then, + // we would do many redundant checks, which would harm performance. + // The only more or less sensible way out might be a new resolver between DateMathExpressioResolver + // and WildcardExpressionResolver, which only checks the non-wildcard index names. if (options.ignoreUnavailable() == false) { IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(expression); if (indexAbstraction == null) { - throw indexNotFoundException(expression); + context.addResolutionError(indexNotFoundException(expression)); } else if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && options.ignoreAliases()) { - throw aliasesNotSupportedException(expression); + context.addResolutionError(aliasesNotSupportedException(expression)); } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.includeDataStreams() == false) { - throw indexNotFoundException(expression); + context.addResolutionError(indexNotFoundException(expression)); } } if (add) { @@ -1060,7 +1152,7 @@ private Set innerResolve(Context context, List expressions, Indi result.removeAll(expand); } if (options.allowNoIndices() == false && matches.isEmpty()) { - throw indexNotFoundException(expression); + context.addResolutionError(indexNotFoundException(expression)); } if (Regex.isSimpleMatchPattern(expression)) { wildcardSeen = true; @@ -1069,13 +1161,13 @@ private Set innerResolve(Context context, List expressions, Indi return result; } - private static void validateAliasOrIndex(String expression) { + private static void validateAliasOrIndex(Context context, String expression) { // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API // does not exist and the path is interpreted as an expression. If the expression begins with an underscore, // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown // if the expression can't be found. if (expression.charAt(0) == '_') { - throw new InvalidIndexNameException(expression, "must not start with '_'."); + context.addResolutionError(new InvalidIndexNameException(expression, "must not start with '_'.")); } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java new file mode 100644 index 0000000000000..1fd9c24b39551 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java @@ -0,0 +1,98 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Collection; +import java.util.Set; +import java.util.function.Predicate; + +/** + * A class that possibly encapsulates resolved indices. See the documentation of {@link ResolvedIndices} for a full + * description. + *

+ * In contrast to ResolvedIndices, objects of this class may convey the message "the resolved indices are unknown". + * This may be for several reasons: + *

    + *
  • The information can be only obtained on a master node
    • + *
  • The particular action does not implement the TransportIndicesResolvingAction interface
    • + *
  • It is infeasible to collect the information
    • + *
+ * For authorization purposes, the case of unknown resolved indices should be usually treated as a "requires + * privileges for all indices" case. + *

+ * The class {@link ResolvedIndices} extends OptionallyResolvedIndices. A safe usage pattern would be thus: + * 

+ *     if (optionallyResolvedIndices instanceof ResolvedIndices resolvedIndices) {
+ *         Set<String;gt names = resolvedIndices.local().names();
+ *     }
+ *
+ */ +@ExperimentalApi +public class OptionallyResolvedIndices { + private static final OptionallyResolvedIndices NOT_PRESENT = new OptionallyResolvedIndices(); + + public static OptionallyResolvedIndices unknown() { + return NOT_PRESENT; + } + + public OptionallyResolvedIndices.Local local() { + return Local.NOT_PRESENT; + } + + /** + * Represents the local (i.e., non-remote) indices referenced by the respective request. + */ + @ExperimentalApi + public static class Local { + private static final OptionallyResolvedIndices.Local NOT_PRESENT = new OptionallyResolvedIndices.Local(); + + /** + * Returns all the local names. These names might be indices, aliases or data streams, depending on the usage. + *

+ * In case this is an isUnknown() object, this will return a set of all concrete indices on the cluster (incl. + * hidden and closed indices). This might be a large object. Be prepared to handle such a large object. + *

+ * This method will be only rarely needed. In most cases ResolvedIndices.names() will be sufficient. + */ + public Set names(ClusterState clusterState) { + return clusterState.metadata().getIndicesLookup().keySet(); + } + + /** + * Returns always true. For the unknown resolved indices, we always assume that these are not empty. + */ + public boolean isEmpty() { + return false; + } + + /** + * Returns always true, as we need to assume that an index is potentially contained if the set of indices is unknown. + */ + public boolean contains(String name) { + return true; + } + + /** + * Returns always true, as we need to assume that an index is potentially contained if the set of indices is unknown. + */ + public boolean containsAny(Collection names) { + return true; + } + + /** + * Returns always true, as we need to assume that an index is potentially contained if the set of indices is unknown. + */ + public boolean containsAny(Predicate namePredicate) { + return true; + } + } +} diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java index 2ec9f9795346a..2be75877b2b55 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java @@ -8,8 +8,10 @@ package org.opensearch.cluster.metadata; +import org.opensearch.action.ActionType; import org.opensearch.action.OriginalIndices; import org.opensearch.cluster.ClusterState; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.index.Index; import org.opensearch.transport.RemoteClusterService; @@ -44,43 +46,49 @@ * just taken without further evaluation * */ -public class ResolvedIndices { - public static ResolvedIndices of(String... indices) { - return new ResolvedIndices( - new Local(Collections.unmodifiableSet(new HashSet<>(Arrays.asList(indices))), null, false), - Remote.EMPTY - ); +@ExperimentalApi +public class ResolvedIndices extends OptionallyResolvedIndices { + public static ResolvedIndices of(String... indicesAliasesAndDataStreams) { + return new ResolvedIndices(new Local(Set.of(indicesAliasesAndDataStreams), null, Map.of()), Remote.EMPTY); } public static ResolvedIndices of(Index... indices) { return new ResolvedIndices( - new Local(Stream.of(indices).map(Index::getName).collect(Collectors.toUnmodifiableSet()), null, false), + new Local.Concrete( + Set.of(indices), + Stream.of(indices).map(Index::getName).collect(Collectors.toUnmodifiableSet()), + null, + Map.of(), + List.of() + ), Remote.EMPTY ); } - public static ResolvedIndices of(Collection indices) { - return new ResolvedIndices(new Local(Collections.unmodifiableSet(new HashSet<>(indices)), null, false), Remote.EMPTY); + public static ResolvedIndices of(Collection indicesAliasesAndDataStreams) { + return new ResolvedIndices(new Local(Set.copyOf(indicesAliasesAndDataStreams), null, Map.of()), Remote.EMPTY); } - public static ResolvedIndices all() { - return ALL; + public static ResolvedIndices of(Local local) { + return new ResolvedIndices(local, Remote.EMPTY); } - public static ResolvedIndices ofNonNull(String... indices) { - Set indexSet = new HashSet<>(indices.length); + public static OptionallyResolvedIndices unknown() { + return OptionallyResolvedIndices.unknown(); + } + + public static ResolvedIndices ofNonNull(String... indicesAliasesAndDataStreams) { + Set indexSet = new HashSet<>(indicesAliasesAndDataStreams.length); - for (String index : indices) { + for (String index : indicesAliasesAndDataStreams) { if (index != null) { indexSet.add(index); } } - return new ResolvedIndices(new Local(Collections.unmodifiableSet(indexSet), null, false), Remote.EMPTY); + return new ResolvedIndices(new Local(Collections.unmodifiableSet(indexSet), null, Map.of()), Remote.EMPTY); } - private static final ResolvedIndices ALL = new ResolvedIndices(new Local(Set.of(Metadata.ALL), null, true), Remote.EMPTY); - private final Local local; private final Remote remote; @@ -89,6 +97,7 @@ private ResolvedIndices(Local local, Remote remote) { this.remote = remote; } + @Override public Local local() { return this.local; } @@ -108,8 +117,16 @@ public ResolvedIndices withRemoteIndices(Map remoteIndi return new ResolvedIndices(this.local, new Remote(Collections.unmodifiableMap(newRemoteIndices))); } + /** + * Returns a ResolvedIndices object associated with the given OriginalIndices object for the local part. This is only for + * convenience, no semantics are implied. + */ public ResolvedIndices withLocalOriginalIndices(OriginalIndices originalIndices) { - return new ResolvedIndices(new Local(this.local.names, originalIndices, this.local.isAll), this.remote); + return new ResolvedIndices(this.local.withOriginalIndices(originalIndices), this.remote); + } + + public ResolvedIndices withLocalSubActions(ActionType actionType, ResolvedIndices.Local local) { + return new ResolvedIndices(this.local.withSubActions(actionType, local), this.remote); } public boolean isEmpty() { @@ -119,44 +136,43 @@ public boolean isEmpty() { /** * Represents the local (i.e., non-remote) indices referenced by the respective request. */ - public static class Local { - private final Set names; - private final OriginalIndices originalIndices; - private final boolean isAll; + @ExperimentalApi + public static class Local extends OptionallyResolvedIndices.Local { + protected final Set names; + protected final OriginalIndices originalIndices; + protected final Map, Local> subActions; + private Set namesOfIndices; + + public static Local of(Collection names) { + return new Local(Set.copyOf(names), null, Map.of()); + } - private Local(Set names, OriginalIndices originalIndices, boolean isAll) { + /** + * Creates a new instance. + *

+ * Note: The caller of this method must make sure that the passed objects are immutable. + * For this reason, this constructor is private. This contract is guaranteed by the static + * constructor methods in this file. + */ + private Local(Set names, OriginalIndices originalIndices, Map, Local> subActions) { this.names = names; this.originalIndices = originalIndices; - this.isAll = isAll; + this.subActions = subActions; } /** * Returns all the local names. These names might be indices, aliases or data streams, depending on the usage. - *

- * Note: You must gate this call by if (!isAll()). - * If isAll() is true, this method will throw an IllegalStateException. If you are sure that you really need all index names, please use the method - * names(ClusterState) instead. * * @return an unmodifiable set of names of indices, aliases and/or data streams. - * @throws IllegalStateException if isAll() is true */ public Set names() { - if (this.isAll) { - throw new IllegalStateException("ResolvedIndices.Local.names() cannot be called for isAll cases"); - } - return this.names; } /** * Returns all the local names. These names might be indices, aliases or data streams, depending on the usage. - *

- * Note: You must gate this call by if (!isAll()). - * If isAll() is true, this method will throw an IllegalStateException. If you are sure that you really need all index names, please use the method - * names(ClusterState) instead. * * @return an array of names of indices, aliases and/or data streams. - * @throws IllegalStateException if isAll() is true */ public String[] namesAsArray() { return this.names().toArray(new String[0]); @@ -165,59 +181,252 @@ public String[] namesAsArray() { /** * Returns all the local names. These names might be indices, aliases or data streams, depending on the usage. *

- * In case this is an isAll() object, this will return a set of all concrete indices on the cluster (incl. + * In case this is an isUnknown() object, this will return a set of all concrete indices on the cluster (incl. * hidden and closed indices). This might be a large object. Be prepared to handle such a large object. *

- * This method will be only rarely needed. In most cases names() will be sufficient. + * This method will be only rarely needed. In most cases ResolvedIndices.names() will be sufficient. */ + @Override public Set names(ClusterState clusterState) { - if (this.isAll) { - return clusterState.metadata().getIndicesLookup().keySet(); - } else { - return this.names; + return this.names; + } + + /** + * Returns all the local names. Any data streams or aliases will be replaced by the member index names. + * In contrast to namesOfConcreteIndices(), this will keep the names of non-existing indices and will never + * throw an exception. + * + * @return an unmodifiable set of index names + */ + public Set namesOfIndices(ClusterState clusterState) { + Set result = this.namesOfIndices; + + if (result == null) { + Map indicesLookup = clusterState.metadata().getIndicesLookup(); + result = new HashSet<>(this.names.size()); + for (String name : this.names) { + IndexAbstraction indexAbstraction = indicesLookup.get(name); + + if (indexAbstraction == null) { + // We keep the names of non existing indices + this.names.add(name); + } else if (indexAbstraction instanceof IndexAbstraction.Index) { + // For normal indices, we just keep its name + this.names.add(name); + } else { + // This is an alias or data stream + for (IndexMetadata index : indexAbstraction.getIndices()) { + result.add(index.getIndex().getName()); + } + } + } + + result = Collections.unmodifiableSet(result); + + this.namesOfIndices = result; } + + return result; } + /** + * Returns any OriginalIndices object associated with this object. + * Note: This is just a convenience method for code that passes around ResolvedIndices objects for managing + * information. This object will be only present if you add it to the object. + */ public OriginalIndices originalIndices() { return this.originalIndices; } + /** + * Sub-actions can be used to specify indices which play a different role in the action processing. + * For example, the swiss-army-knife IndicesAliases action can delete indices. The subActions() property + * can be used to specify indices with such special roles. + */ + public Map, Local> subActions() { + return this.subActions; + } + + /** + * Returns true if there are no local indices. + */ + @Override public boolean isEmpty() { - if (this.isAll) { - return false; - } else { - return this.names.isEmpty(); - } + return this.names.isEmpty(); } - public boolean isAll() { - return this.isAll; + /** + * Returns true if the local names contain an entry with the given name. + */ + @Override + public boolean contains(String name) { + return this.names.contains(name); } - public boolean contains(String index) { - if (this.isAll) { - return true; - } else { - return this.names.contains(index); - } + /** + * Returns true if the local names contain any of the specified names. + */ + @Override + public boolean containsAny(Collection names) { + return names.stream().anyMatch(this.names::contains); } - public boolean containsAny(Collection indices) { - if (this.isAll) { - return true; - } else { - return indices.stream().anyMatch(this.names::contains); - } + /** + * Returns true if any of the local names match the given predicate. + */ + @Override + public boolean containsAny(Predicate namePredicate) { + return this.names.stream().anyMatch(namePredicate); + } + + /** + * Returns a ResolvedIndices.Local object associated with the given OriginalIndices object. This is only for + * convenience, no semantics are implied. + */ + public ResolvedIndices.Local withOriginalIndices(OriginalIndices originalIndices) { + return new Local(this.names, originalIndices, this.subActions); } - public boolean containsAny(Predicate indexNamePredicate) { - return this.names.stream().anyMatch(indexNamePredicate); + public ResolvedIndices.Local withSubActions(ActionType actionType, ResolvedIndices.Local local) { + Map, Local> subActions = new HashMap<>(this.subActions); + subActions.put(actionType, local); + return new Local(this.names, this.originalIndices, Collections.unmodifiableMap(subActions)); + } + + /** + * This is a specialization of the {@link ResolvedIndices.Local} class which additionally + * carries {@link Index} objects. The {@link IndexNameExpressionResolver} produces such objects. + *

+ * Important: The methods that give access to the concrete indices can throw + * exceptions such as {@link org.opensearch.index.IndexNotFoundException} if the concrete indices + * could not be determined. The methods from the Local super class, such as names(), still give + * access to index information. + */ + @ExperimentalApi + public static class Concrete extends Local { + private static final Concrete EMPTY = new Concrete(Set.of(), Set.of(), null, Map.of(), List.of()); + + public static Concrete empty() { + return EMPTY; + } + + public static Concrete of(Index... concreteIndices) { + return new Concrete( + Set.of(concreteIndices), + Stream.of(concreteIndices).map(Index::getName).collect(Collectors.toSet()), + null, + Map.of(), + List.of() + ); + } + + /** + * Creates a new RemoteIndices.Local.Concrete object with the given concrete indices and names. + * This is primarily used by IndexNameExpressionResolver to construct return values, that's why it is + * package private. There may be more names than concrete indices, for example when a referenced index does + * not exist. Thus, names should be usually a super set of concreteIndices. This method does not verify + * that, it is the duty of the caller to make this sure. + */ + static Concrete of(Set concreteIndices, Set names) { + return new Concrete(Set.copyOf(concreteIndices), Set.copyOf(names), null, Map.of(), List.of()); + } + + private final Set concreteIndices; + private final List resolutionErrors; + + private Concrete( + Set concreteIndices, + Set names, + OriginalIndices originalIndices, + Map, Local> subActions, + List resolutionErrors + ) { + super(names, originalIndices, subActions); + this.concreteIndices = concreteIndices; + this.resolutionErrors = resolutionErrors; + } + + /** + * Returns the concrete indices. This might throw an exception if there were issues during index resolution. + * If you need access to index information while avoiding exceptions, use the names() method instead. + * + * @throws org.opensearch.index.IndexNotFoundException This exception is thrown for several conditions: + * 1. If one of the index expression pointed to a missing index, alias or data stream and the IndicesOptions + * used during resolution do not allow such a case. 2. If the set of resolved concrete indices is empty and + * the IndicesOptions used during resolution do not allow such a case. + * @throws IllegalArgumentException if one of the aliases resolved to multiple indices and + * IndicesOptions.FORBID_ALIASES_TO_MULTIPLE_INDICES was set. + */ + public Set concreteIndices() { + checkResolutionErrors(); + return this.concreteIndices; + } + + public Index[] concreteIndicesAsArray() { + return this.concreteIndices().toArray(Index.EMPTY_ARRAY); + } + + public Set namesOfConcreteIndices() { + return this.concreteIndices().stream().map(Index::getName).collect(Collectors.toSet()); + } + + public String[] namesOfConcreteIndicesAsArray() { + return this.concreteIndices().stream().map(Index::getName).toArray(String[]::new); + } + + public ResolvedIndices.Local.Concrete withOriginalIndices(OriginalIndices originalIndices) { + return new Concrete(this.concreteIndices, this.names, originalIndices, this.subActions, resolutionErrors); + } + + public ResolvedIndices.Local withSubActions(ActionType actionType, ResolvedIndices.Local local) { + Map, Local> subActions = new HashMap<>(this.subActions); + subActions.put(actionType, local); + return new Concrete(this.concreteIndices, this.names, this.originalIndices, subActions, resolutionErrors); + } + + public ResolvedIndices.Local.Concrete withResolutionErrors(List resolutionErrors) { + if (resolutionErrors.isEmpty()) { + return this; + } else { + return new Concrete( + this.concreteIndices, + this.names(), + originalIndices, + this.subActions, + Stream.concat(this.resolutionErrors.stream(), resolutionErrors.stream()).toList() + ); + } + } + + public ResolvedIndices.Local.Concrete withResolutionErrors(RuntimeException... resolutionErrors) { + return withResolutionErrors(Arrays.asList(resolutionErrors)); + } + + public ResolvedIndices.Local.Concrete withoutResolutionErrors() { + return new Concrete(this.concreteIndices, this.names(), this.originalIndices, this.subActions, List.of()); + } + + List resolutionErrors() { + return this.resolutionErrors; + } + + private void checkResolutionErrors() { + if (!this.resolutionErrors.isEmpty()) { + throw this.resolutionErrors.getFirst(); + } + } + + @Override + public String toString() { + return "{" + "concreteIndices=" + concreteIndices + ", names=" + names() + ", resolutionErrors=" + resolutionErrors + '}'; + } } } /** * Represents the remote indices part of the respective request. */ + @ExperimentalApi public static class Remote { static final Remote EMPTY = new Remote(Collections.emptyMap(), Collections.emptyList()); diff --git a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java index 75bcf0b9f8d98..480c979b6627a 100644 --- a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java +++ b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java @@ -81,12 +81,8 @@ public static boolean matchesPluginSystemIndexPattern(String pluginClassName, St if (systemIndexDescriptors == null) { return false; } - String[] pluginSystemIndexPatterns = systemIndexDescriptors - .stream() - .map(SystemIndexDescriptor::getIndexPattern) - .toArray(String[]::new); - - + return systemIndexDescriptors.stream() + .anyMatch(systemIndexDescriptor -> Regex.simpleMatch(systemIndexDescriptor.getIndexPattern(), index)); } static List getAllDescriptors() { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java index c2caae1dc2586..8c54b195711af 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -35,6 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.IndicesRequest; +import org.opensearch.action.OriginalIndices; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.delete.DeleteRequest; @@ -817,6 +818,27 @@ public void testConcreteIndicesIgnoreIndicesOneMissingIndex() { assertThat(infe.getMessage(), is("no such index [testZZZ]")); } + /** + * Same as testConcreteIndicesIgnoreIndicesOneMissingIndex() but using the concreteResolvedIndices() method. + * This should keep the name of the missing index and defer the throwing of the exception to the point + * when ResolvedIndices.Local.Concrete.concreteIndices() is called. + */ + public void testConcreteResolvedIndicesIgnoreIndicesOneMissingIndex() { + Metadata.Builder mdBuilder = Metadata.builder().put(indexBuilder("testXXX")).put(indexBuilder("kuku")); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context( + state, + IndicesOptions.strictExpandOpen(), + false + ); + + ResolvedIndices.Local.Concrete concreteResolvedIndices = indexNameExpressionResolver.concreteResolvedIndices(context, "testZZZ"); + assertEquals(concreteResolvedIndices.names(), Set.of("testZZZ")); + + IndexNotFoundException infe = expectThrows(IndexNotFoundException.class, () -> concreteResolvedIndices.concreteIndices()); + assertThat(infe.getMessage(), is("no such index [testZZZ]")); + } + public void testConcreteIndicesIgnoreIndicesOneMissingIndexOtherFound() { Metadata.Builder mdBuilder = Metadata.builder().put(indexBuilder("testXXX")).put(indexBuilder("kuku")); ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); @@ -2268,8 +2290,13 @@ public void testDataStreams() { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream"); assertThat(result.length, equalTo(2)); - assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); - assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2))); + assertThat( + Arrays.stream(result).map(Index::getName).collect(Collectors.toList()), + containsInAnyOrder( + DataStream.getDefaultBackingIndexName(dataStreamName, 1), + DataStream.getDefaultBackingIndexName(dataStreamName, 2) + ) + ); } { // Ignore data streams @@ -2521,6 +2548,87 @@ public void testDataStreamsNames() { assertThat(names, empty()); } + public void testConcreteResolvedIndicesWithWildcards() { + Metadata.Builder mdBuilder = Metadata.builder() + .put(indexBuilder("index_a1").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_a2").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_b1")) + .put(indexBuilder("index_b2")); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + + IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context( + state, + IndicesOptions.lenientExpandOpen(), + false + ); + + ResolvedIndices.Local.Concrete concreteResolvedIndices = indexNameExpressionResolver.concreteResolvedIndices( + context, + "alias_a", + "index_b*", + "index_c*", + "index_d" + ); + + // We test the following semantics: + // - Of course, the concrete indices will contain all existing indices that are matched. + // - For the names() property we include: + // - resolved index patterns + // - names of existing indices + // - names of non-existing indices + assertThat(concreteResolvedIndices.names(), containsInAnyOrder("alias_a", "index_b1", "index_b2", "index_d")); + assertThat(concreteResolvedIndices.namesOfConcreteIndices(), containsInAnyOrder("index_a1", "index_a2", "index_b1", "index_b2")); + } + + public void testConcreteResolvedIndicesWithErrors() { + Metadata.Builder mdBuilder = Metadata.builder() + .put(indexBuilder("index_a1").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_a2").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_closed").state(IndexMetadata.State.CLOSE)) + .put(indexBuilder("index_b1").putAlias(AliasMetadata.builder("alias_b"))) + .put(indexBuilder("index_c1")); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + + IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context( + state, + IndicesOptions.strictSingleIndexNoExpandForbidClosed(), + false + ); + + ResolvedIndices.Local.Concrete concreteResolvedIndices = indexNameExpressionResolver.concreteResolvedIndices( + context, + "baz*", + "alias_a", + "index_closed", + "alias_b", + "index_c1" + ); + assertThat(concreteResolvedIndices.names(), containsInAnyOrder("index_closed", "index_c1", "baz*", "alias_b", "alias_a")); + assertThat(concreteResolvedIndices.withoutResolutionErrors().namesOfConcreteIndices(), containsInAnyOrder("index_b1", "index_c1")); + assertThat( + concreteResolvedIndices.resolutionErrors().stream().map(Throwable::getMessage).toList(), + containsInAnyOrder( + "no such index [baz*]", + "alias [alias_a] has more than one index associated with it [index_a1, index_a2], can't execute a single index op", + "closed" + ) + ); + } + + public void testConcreteResolvedIndicesWithIndicesRequestParam() { + Metadata.Builder mdBuilder = Metadata.builder().put(indexBuilder("index_b1")).put(indexBuilder("index_b2")); + ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + + ResolvedIndices.Local.Concrete concreteResolvedIndices = indexNameExpressionResolver.concreteResolvedIndices( + state, + new OriginalIndices(new String[] { "index_b*", "index_d" }, IndicesOptions.lenientExpandOpen()), + 0 + ); + + assertThat(concreteResolvedIndices.names(), containsInAnyOrder("index_b1", "index_b2", "index_d")); + assertThat(concreteResolvedIndices.namesOfConcreteIndices(), containsInAnyOrder("index_b1", "index_b2")); + } + static class FakeExpressionResolver implements IndexNameExpressionResolver.ExpressionResolver { @Override public List resolve(IndexNameExpressionResolver.Context context, List expressions) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java new file mode 100644 index 0000000000000..ad2ec849b291e --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.action.OriginalIndices; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.core.index.Index; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isA; + +public class ResolvedIndicesTests extends OpenSearchTestCase { + + public void testOfNames() { + ResolvedIndices resolvedIndices = ResolvedIndices.of("a", "b", "c"); + assertThat(resolvedIndices.local().names(), containsInAnyOrder("a", "b", "c")); + assertThat(Arrays.asList(resolvedIndices.local().namesAsArray()), containsInAnyOrder("a", "b", "c")); + assertFalse(resolvedIndices.local().isEmpty()); + assertTrue(resolvedIndices.remote().isEmpty()); + assertFalse(resolvedIndices.isEmpty()); + } + + public void testOfConcreteIndices() { + ResolvedIndices resolvedIndices = ResolvedIndices.of(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b")); + assertThat(resolvedIndices.local().names(), containsInAnyOrder("index_a", "index_b")); + assertThat(resolvedIndices.local(), isA(ResolvedIndices.Local.Concrete.class)); + ResolvedIndices.Local.Concrete concrete = (ResolvedIndices.Local.Concrete) resolvedIndices.local(); + assertThat(concrete.concreteIndices(), containsInAnyOrder(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b"))); + } + + public void testOfNonNull() { + ResolvedIndices resolvedIndices = ResolvedIndices.ofNonNull("a", "b", "c", null); + assertThat(resolvedIndices.local().names(), containsInAnyOrder("a", "b", "c")); + } + + public void testWithLocalOriginalIndices() { + OriginalIndices originalIndices = new OriginalIndices(new String[] { "x", "y" }, IndicesOptions.LENIENT_EXPAND_OPEN); + ResolvedIndices resolvedIndices = ResolvedIndices.of("a", "b", "c").withLocalOriginalIndices(originalIndices); + assertThat(resolvedIndices.local().names(), containsInAnyOrder("a", "b", "c")); + assertArrayEquals(resolvedIndices.local().originalIndices().indices(), new String[] { "x", "y" }); + } + + public void testOfEmpty() { + ResolvedIndices resolvedIndices = ResolvedIndices.of(new String[0]); + assertTrue(resolvedIndices.local().isEmpty()); + assertTrue(resolvedIndices.remote().isEmpty()); + assertTrue(resolvedIndices.isEmpty()); + } + + public void testWithRemoteIndices() { + Map remoteIndices = Map.of( + "remote_cluster", + new OriginalIndices(new String[] { "remote_index" }, IndicesOptions.LENIENT_EXPAND_OPEN) + ); + ResolvedIndices resolvedIndices = ResolvedIndices.of(new String[0]).withRemoteIndices(remoteIndices); + assertTrue(resolvedIndices.remote().asClusterToOriginalIndicesMap().containsKey("remote_cluster")); + assertArrayEquals( + new String[] { "remote_index" }, + resolvedIndices.remote().asClusterToOriginalIndicesMap().get("remote_cluster").indices() + ); + assertThat(resolvedIndices.remote().asRawExpressions(), containsInAnyOrder("remote_cluster:remote_index")); + } + + public void testLocalConcreteOf() { + Set indices = Set.of(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b")); + Set names = Set.of("index_a", "index_b", "index_c"); + ResolvedIndices.Local.Concrete concrete = ResolvedIndices.Local.Concrete.of(indices, names).withResolutionErrors(); + assertThat(concrete.concreteIndices(), containsInAnyOrder(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b"))); + assertThat(concrete.namesOfConcreteIndices(), containsInAnyOrder("index_a", "index_b")); + assertThat(Arrays.asList(concrete.namesOfConcreteIndicesAsArray()), containsInAnyOrder("index_a", "index_b")); + assertThat(concrete.names(), containsInAnyOrder("index_a", "index_b", "index_c")); + } + + public void testLocalConcreteWithResolutionErrors() { + Set indices = Set.of(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b")); + Set names = Set.of("index_a", "index_b", "index_c"); + ResolvedIndices.Local.Concrete concrete = ResolvedIndices.Local.Concrete.of(indices, names) + .withResolutionErrors(new IndexNotFoundException("index_x")); + assertThat(concrete.names(), containsInAnyOrder("index_a", "index_b", "index_c")); + IndexNotFoundException exception = expectThrows(IndexNotFoundException.class, concrete::concreteIndices); + assertThat(exception.getIndex().getName(), equalTo("index_x")); + concrete = concrete.withoutResolutionErrors(); + assertThat(concrete.concreteIndices(), containsInAnyOrder(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b"))); + } + +} From daddcff63715302da3ffc7bbd5bb7804a21e88ba Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Thu, 28 Aug 2025 18:28:26 +0200 Subject: [PATCH 05/24] Fix Signed-off-by: Nils Bandener --- .../admin/indices/mapping/put/PutMappingRequestTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index 377e2bd0c9397..201fa6c087364 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -184,7 +184,7 @@ public void testResolveIndicesWithWriteIndexOnlyAndDataStreamsAndWriteAliases() cs, request, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) - ); + ).concreteIndicesAsArray(); List indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList()); IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo"); // should resolve the data stream and each alias to their respective write indices @@ -212,7 +212,7 @@ public void testResolveIndicesWithoutWriteIndexOnlyAndDataStreamsAndWriteAliases cs, request, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) - ); + ).concreteIndicesAsArray(); List indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList()); IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo"); List expectedIndices = expectedDs.getIndices().stream().map(im -> im.getIndex().getName()).collect(Collectors.toList()); @@ -242,7 +242,7 @@ public void testResolveIndicesWithWriteIndexOnlyAndDataStreamAndIndex() { cs, request, new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) - ); + ).concreteIndicesAsArray(); List indexNames = Arrays.stream(indices).map(Index::getName).collect(Collectors.toList()); IndexAbstraction expectedDs = cs.metadata().getIndicesLookup().get("foo"); List expectedIndices = expectedDs.getIndices().stream().map(im -> im.getIndex().getName()).collect(Collectors.toList()); From f7d2398d893c212b4f1cb5d24858680845ecc02a Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 29 Aug 2025 03:03:43 +0200 Subject: [PATCH 06/24] Fixes Signed-off-by: Nils Bandener --- .../TransportFieldCapabilitiesAction.java | 2 +- .../metadata/IndexNameExpressionResolver.java | 20 ++++++++++++- .../cluster/metadata/ResolvedIndices.java | 4 +-- .../indices/get/GetIndexActionTests.java | 6 ++++ .../put/TransportPutMappingActionTests.java | 2 ++ .../WildcardExpressionResolverTests.java | 29 +++++++++++-------- 6 files changed, 47 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index f9b9691d8b307..8ee24f6acf281 100644 --- a/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/opensearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -95,7 +95,7 @@ protected void doExecute(Task task, FieldCapabilitiesRequest request, final Acti // retrieve the initial timestamp in case the action is a cross cluster search long nowInMillis = request.nowInMillis() == null ? System.currentTimeMillis() : request.nowInMillis(); ResolvedIndices allResolvedIndices = resolveIndices(request, clusterService.state()); - Set concreteIndices = allResolvedIndices.local().names(); + Set concreteIndices = ((ResolvedIndices.Local.Concrete) allResolvedIndices.local()).namesOfConcreteIndices(); Map remoteClusterIndices = allResolvedIndices.remote().asClusterToOriginalIndicesMap(); OriginalIndices localIndices = allResolvedIndices.local().originalIndices(); final int totalNumRequest = concreteIndices.size() + remoteClusterIndices.size(); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java index d9a1e66919fec..3ea2cf97ef358 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java @@ -207,6 +207,10 @@ public List dataStreamNames(ClusterState state, IndicesOptions options, List dataStreams = wildcardExpressionResolver.resolve(context, finalExpressions); + if (!context.resolutionErrors.isEmpty()) { + throw context.resolutionErrors.getFirst(); + } + return ((dataStreams == null) ? List.of() : dataStreams).stream() .map(x -> state.metadata().getIndicesLookup().get(x)) .filter(Objects::nonNull) @@ -613,6 +617,9 @@ public Set resolveExpressions(ClusterState state, String... expressions) for (ExpressionResolver expressionResolver : expressionResolvers) { resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions); } + if (!context.resolutionErrors.isEmpty()) { + throw context.resolutionErrors.getFirst(); + } return Collections.unmodifiableSet(new HashSet<>(resolvedExpressions)); } @@ -707,6 +714,9 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab for (ExpressionResolver expressionResolver : expressionResolvers) { resolvedExpressions = expressionResolver.resolve(context, resolvedExpressions); } + if (!context.resolutionErrors.isEmpty()) { + throw context.resolutionErrors.getFirst(); + } // TODO: it appears that this can never be true? if (isAllIndices(resolvedExpressions)) { @@ -1013,9 +1023,17 @@ public void addResolutionError(RuntimeException resolutionError) { * Note: This is on purpose package private. External resolvers should have no need to inspect the * present resolution errors. */ - private List getResolutionErrors() { + List getResolutionErrors() { return this.resolutionErrors; } + + RuntimeException getFirstResolutionError() { + if (this.resolutionErrors.isEmpty()) { + return null; + } else { + return this.resolutionErrors.getFirst(); + } + } } /** diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java index 2be75877b2b55..6950d741cf756 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java @@ -209,10 +209,10 @@ public Set namesOfIndices(ClusterState clusterState) { if (indexAbstraction == null) { // We keep the names of non existing indices - this.names.add(name); + result.add(name); } else if (indexAbstraction instanceof IndexAbstraction.Index) { // For normal indices, we just keep its name - this.names.add(name); + result.add(name); } else { // This is an alias or data stream for (IndexMetadata index : indexAbstraction.getIndices()) { diff --git a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java index 67d2163affd28..29f3698fdb2fa 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java @@ -38,6 +38,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Context; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; @@ -199,5 +200,10 @@ public Index[] concreteIndices(ClusterState state, IndicesRequest request) { } return out; } + + @Override + public ResolvedIndices.Local.Concrete concreteResolvedIndices(ClusterState state, IndicesRequest request) { + return ResolvedIndices.Local.Concrete.of(concreteIndices(state, request)); + } } } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingActionTests.java index f1ed0a09230bf..dc9bd6b0bdb9d 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingActionTests.java @@ -15,6 +15,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataMappingService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.mapper.MappingTransformerRegistry; @@ -66,6 +67,7 @@ public void setup() { MockitoAnnotations.openMocks(this); ActionFilter[] emptyActionFilters = new ActionFilter[] {}; + when(indexNameExpressionResolver.concreteResolvedIndices(any(), any())).thenReturn(ResolvedIndices.Local.Concrete.empty()); when(actionFilters.filters()).thenReturn(emptyActionFilters); action = new TransportPutMappingAction( transportService, diff --git a/server/src/test/java/org/opensearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/opensearch/cluster/metadata/WildcardExpressionResolverTests.java index 03807fb5f8c4d..bd698990dc5cc 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/WildcardExpressionResolverTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/WildcardExpressionResolverTests.java @@ -233,11 +233,6 @@ public void testResolveAliases() { ); // ignoreAliases option is set, WildcardExpressionResolver resolves the provided expressions only against the defined indices IndicesOptions errorOnAliasIndicesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false); - IndexNameExpressionResolver.Context skipAliasesStrictContext = new IndexNameExpressionResolver.Context( - state, - errorOnAliasIndicesOptions, - false - ); { List indices = resolver.resolve(indicesAndAliasesContext, Collections.singletonList("foo_a*")); @@ -248,11 +243,13 @@ public void testResolveAliases() { assertEquals(0, indices.size()); } { - IndexNotFoundException infe = expectThrows( - IndexNotFoundException.class, - () -> resolver.resolve(skipAliasesStrictContext, Collections.singletonList("foo_a*")) + IndexNameExpressionResolver.Context skipAliasesStrictContext = new IndexNameExpressionResolver.Context( + state, + errorOnAliasIndicesOptions, + false ); - assertEquals("foo_a*", infe.getIndex().getName()); + resolver.resolve(skipAliasesStrictContext, Collections.singletonList("foo_a*")); + assertEquals("foo_a*", ((IndexNotFoundException) skipAliasesStrictContext.getFirstResolutionError()).getIndex().getName()); } { List indices = resolver.resolve(indicesAndAliasesContext, Collections.singletonList("foo*")); @@ -263,6 +260,11 @@ public void testResolveAliases() { assertThat(indices, containsInAnyOrder("foo_foo", "foo_index")); } { + IndexNameExpressionResolver.Context skipAliasesStrictContext = new IndexNameExpressionResolver.Context( + state, + errorOnAliasIndicesOptions, + false + ); List indices = resolver.resolve(skipAliasesStrictContext, Collections.singletonList("foo*")); assertThat(indices, containsInAnyOrder("foo_foo", "foo_index")); } @@ -275,10 +277,13 @@ public void testResolveAliases() { assertThat(indices, containsInAnyOrder("foo_alias")); } { - IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> resolver.resolve(skipAliasesStrictContext, Collections.singletonList("foo_alias")) + IndexNameExpressionResolver.Context skipAliasesStrictContext = new IndexNameExpressionResolver.Context( + state, + errorOnAliasIndicesOptions, + false ); + resolver.resolve(skipAliasesStrictContext, Collections.singletonList("foo_alias")); + IllegalArgumentException iae = (IllegalArgumentException) skipAliasesStrictContext.getFirstResolutionError(); assertEquals( "The provided expression [foo_alias] matches an alias, " + "specify the corresponding concrete indices instead.", iae.getMessage() From 5eb7a4a50788c3fe07950583d571d2a1f763d315 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 29 Aug 2025 05:42:59 +0200 Subject: [PATCH 07/24] Fixes Signed-off-by: Nils Bandener --- .../action/support/broadcast/TransportBroadcastAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java index fe4449516be86..47ac4edeff86f 100644 --- a/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java +++ b/server/src/main/java/org/opensearch/action/support/broadcast/TransportBroadcastAction.java @@ -167,7 +167,7 @@ protected AsyncBroadcastAction(Task task, Request request, ActionListener Date: Fri, 29 Aug 2025 10:53:13 +0200 Subject: [PATCH 08/24] Fixes Signed-off-by: Nils Bandener --- .../action/bulk/TransportBulkAction.java | 11 ++++++++++- .../metadata/OptionallyResolvedIndices.java | 5 +++++ .../cluster/metadata/ResolvedIndices.java | 19 +++++++++++++++++++ .../indices/SystemIndexRegistry.java | 15 ++++++++++----- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java index 79d523c6774bc..02f72d55df1c8 100644 --- a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java @@ -51,6 +51,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.AutoCreateIndex; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.update.TransportUpdateAction; import org.opensearch.action.update.UpdateRequest; import org.opensearch.action.update.UpdateResponse; @@ -64,6 +65,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.ValidationException; import org.opensearch.common.inject.Inject; @@ -123,7 +125,9 @@ * * @opensearch.internal */ -public class TransportBulkAction extends HandledTransportAction { +public class TransportBulkAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private static final Logger logger = LogManager.getLogger(TransportBulkAction.class); @@ -552,6 +556,11 @@ private long buildTookInMillis(long startTimeNanos) { return TimeUnit.NANOSECONDS.toMillis(relativeTime() - startTimeNanos); } + @Override + public ResolvedIndices resolveIndices(BulkRequest request) { + return ResolvedIndices.of(request.getIndices()); + } + /** * retries on retryable cluster blocks, resolves item requests, * constructs shard bulk requests and delegates execution to shard bulk action diff --git a/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java index 1fd9c24b39551..18a359b38c0b7 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java @@ -48,6 +48,11 @@ public OptionallyResolvedIndices.Local local() { return Local.NOT_PRESENT; } + @Override + public String toString() { + return "ResolvedIndices{unknown=true}"; + } + /** * Represents the local (i.e., non-remote) indices referenced by the respective request. */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java index 6950d741cf756..c643a42591ee4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java @@ -133,6 +133,11 @@ public boolean isEmpty() { return this.local.isEmpty() && this.remote.isEmpty(); } + @Override + public String toString() { + return "ResolvedIndices{" + "local=" + local + ", remote=" + remote + '}'; + } + /** * Represents the local (i.e., non-remote) indices referenced by the respective request. */ @@ -293,6 +298,15 @@ public ResolvedIndices.Local withSubActions(ActionType actionType, ResolvedIn return new Local(this.names, this.originalIndices, Collections.unmodifiableMap(subActions)); } + @Override + public String toString() { + if (this.subActions.isEmpty()) { + return "{names=" + names() + "}"; + } else { + return "{names=" + names() + ", subActions=" + subActions + '}'; + } + } + /** * This is a specialization of the {@link ResolvedIndices.Local} class which additionally * carries {@link Index} objects. The {@link IndexNameExpressionResolver} produces such objects. @@ -464,6 +478,11 @@ public boolean isEmpty() { return this.clusterToOriginalIndicesMap.isEmpty(); } + @Override + public String toString() { + return this.asRawExpressions().toString(); + } + private List buildRawExpressions() { if (this.clusterToOriginalIndicesMap.isEmpty()) { return Collections.emptyList(); diff --git a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java index 480c979b6627a..7f8068e54c52d 100644 --- a/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java +++ b/server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import static java.util.Collections.singletonList; @@ -76,13 +77,17 @@ public static Set matchesPluginSystemIndexPattern(String pluginClassName .collect(Collectors.toSet()); } - public static boolean matchesPluginSystemIndexPattern(String pluginClassName, String index) { + /** + * Returns a predicate that can be used to test if an index is registered as system index for the given plugin. + */ + public static Predicate getPluginSystemIndexPredicate(String pluginClassName) { Collection systemIndexDescriptors = SYSTEM_INDEX_DESCRIPTORS_MAP.get(pluginClassName); - if (systemIndexDescriptors == null) { - return false; + if (systemIndexDescriptors == null || systemIndexDescriptors.isEmpty()) { + return index -> false; + } else { + return index -> systemIndexDescriptors.stream() + .anyMatch(systemIndexDescriptor -> Regex.simpleMatch(systemIndexDescriptor.getIndexPattern(), index)); } - return systemIndexDescriptors.stream() - .anyMatch(systemIndexDescriptor -> Regex.simpleMatch(systemIndexDescriptor.getIndexPattern(), index)); } static List getAllDescriptors() { From d23dd35521a15fc556c934e324e7166e477d5703 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 3 Sep 2025 08:58:12 +0200 Subject: [PATCH 09/24] Tests Signed-off-by: Nils Bandener --- .../alias/get/TransportGetAliasesAction.java | 4 + .../metadata/OptionallyResolvedIndices.java | 14 +++ .../cluster/metadata/ResolvedIndices.java | 64 ++++++++++ .../TransportIndicesAliasesActionTests.java | 114 ++++++++++++++++++ .../get/TransportGetAliasesActionTests.java | 39 ++++++ .../indices/create/AutoCreateActionTests.java | 47 ++++++++ .../datastream/GetDataStreamActionTests.java | 61 ++++++++++ .../search/TransportDeletePitActionTests.java | 86 +++++++++++++ .../support/ActionRequestMetadataTests.java | 85 +++++++++++++ .../TransportBroadcastByNodeActionTests.java | 8 ++ ...ortInstanceSingleOperationActionTests.java | 18 +++ .../IndexNameExpressionResolverTests.java | 11 ++ .../metadata/ResolvedIndicesTests.java | 71 +++++++++++ .../indices/SystemIndicesTests.java | 16 +++ 14 files changed, 638 insertions(+) create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesActionTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamActionTests.java create mode 100644 server/src/test/java/org/opensearch/action/support/ActionRequestMetadataTests.java diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index c57cf75cbc263..a3807cee6efd9 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -217,6 +217,10 @@ public ResolvedIndices resolveIndices(GetAliasesRequest request) { // which will be resolved by state.metadata().findAliases() to all aliases, but limited to the aliases // containing one of the specified indices // - both aliases and indices specified: this is then the intersection + // + // The consequence for the resolveIndices() method is that the semantics of the return value might be debatable. + // We resort to just the index names, because it is the most precise dimension. If we would include alias names, + // these could also refer to indices which were not requested. String[] concreteIndices; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java index 18a359b38c0b7..286d871e06e74 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/OptionallyResolvedIndices.java @@ -53,6 +53,20 @@ public String toString() { return "ResolvedIndices{unknown=true}"; } + @Override + public boolean equals(Object other) { + if (!(other instanceof OptionallyResolvedIndices otherResolvedIndices)) { + return false; + } + + return this.local() == otherResolvedIndices.local(); + } + + @Override + public int hashCode() { + return 92; + } + /** * Represents the local (i.e., non-remote) indices referenced by the respective request. */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java index c643a42591ee4..51db46c94e3fd 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java @@ -138,6 +138,20 @@ public String toString() { return "ResolvedIndices{" + "local=" + local + ", remote=" + remote + '}'; } + @Override + public boolean equals(Object other) { + if (!(other instanceof ResolvedIndices otherResolvedIndices)) { + return false; + } + + return this.local.equals(otherResolvedIndices.local) && this.remote.equals(otherResolvedIndices.remote); + } + + @Override + public int hashCode() { + return this.local.hashCode() + this.remote.hashCode(); + } + /** * Represents the local (i.e., non-remote) indices referenced by the respective request. */ @@ -152,6 +166,10 @@ public static Local of(Collection names) { return new Local(Set.copyOf(names), null, Map.of()); } + public static Local of(String... names) { + return of(Arrays.asList(names)); + } + /** * Creates a new instance. *

@@ -307,6 +325,20 @@ public String toString() { } } + @Override + public boolean equals(Object other) { + if (!(other instanceof ResolvedIndices.Local otherLocal)) { + return false; + } + + return this.names.equals(otherLocal.names) && this.subActions.equals(otherLocal.subActions); + } + + @Override + public int hashCode() { + return this.names.hashCode() + this.subActions.hashCode(); + } + /** * This is a specialization of the {@link ResolvedIndices.Local} class which additionally * carries {@link Index} objects. The {@link IndexNameExpressionResolver} produces such objects. @@ -420,6 +452,24 @@ public ResolvedIndices.Local.Concrete withoutResolutionErrors() { return new Concrete(this.concreteIndices, this.names(), this.originalIndices, this.subActions, List.of()); } + @Override + public boolean equals(Object other) { + if (!super.equals(other)) { + return false; + } + + if (!(other instanceof ResolvedIndices.Local.Concrete otherLocal)) { + return false; + } + + return this.concreteIndices.equals(otherLocal.concreteIndices); + } + + @Override + public int hashCode() { + return super.hashCode() + this.concreteIndices.hashCode() * 11; + } + List resolutionErrors() { return this.resolutionErrors; } @@ -483,6 +533,20 @@ public String toString() { return this.asRawExpressions().toString(); } + @Override + public boolean equals(Object other) { + if (!(other instanceof ResolvedIndices.Remote otherRemote)) { + return false; + } + + return this.asRawExpressions().equals(otherRemote.asRawExpressions()); + } + + @Override + public int hashCode() { + return this.asRawExpressions().hashCode(); + } + private List buildRawExpressions() { if (this.clusterToOriginalIndicesMap.isEmpty()) { return Collections.emptyList(); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesActionTests.java new file mode 100644 index 0000000000000..fa703b4ea720e --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesActionTests.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.alias; + +import org.opensearch.Version; +import org.opensearch.action.RequestValidators; +import org.opensearch.action.admin.indices.delete.DeleteIndexAction; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.MetadataIndexAliasesService; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportIndicesAliasesActionTests extends OpenSearchTestCase { + private static ThreadPool threadPool; + private ClusterService clusterService; + private TransportIndicesAliasesAction subject; + + @BeforeClass + public static void beforeClass() { + threadPool = new TestThreadPool(getTestClass().getName()); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + Metadata.Builder mdBuilder = Metadata.builder() + .put(indexBuilder("index_a1").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_a2").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_b1")) + .put(indexBuilder("index_b2")); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + + this.clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + + this.subject = new TransportIndicesAliasesAction( + mock(TransportService.class), + clusterService, + threadPool, + mock(MetadataIndexAliasesService.class), + mock(ActionFilters.class), + new IndexNameExpressionResolver(threadPool.getThreadContext()), + new RequestValidators<>(List.of()) + ); + } + + @AfterClass + public static void afterClass() { + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + threadPool = null; + } + + public void testResolvedIndices_addAliasAction() { + ResolvedIndices resolvedIndices = subject.resolveIndices( + new IndicesAliasesRequest().addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index_b*").alias("alias_b")) + ); + assertEquals(ResolvedIndices.of("index_b1", "index_b2", "alias_b"), resolvedIndices); + } + + public void testResolvedIndices_removeAliasAction() { + ResolvedIndices resolvedIndices = subject.resolveIndices( + new IndicesAliasesRequest().addAliasAction(IndicesAliasesRequest.AliasActions.remove().index("index_a*").alias("alias_*")) + ); + assertEquals(ResolvedIndices.of("index_a1", "index_a2", "alias_a"), resolvedIndices); + } + + public void testResolvedIndices_removeIndexAction() { + ResolvedIndices resolvedIndices = subject.resolveIndices( + new IndicesAliasesRequest().addAliasAction(IndicesAliasesRequest.AliasActions.removeIndex().index("index_a*")) + ); + assertEquals( + ResolvedIndices.of(Collections.emptyList()) + .withLocalSubActions(DeleteIndexAction.INSTANCE, ResolvedIndices.Local.of("index_a1", "index_a2")), + resolvedIndices + ); + } + + private static IndexMetadata.Builder indexBuilder(String index) { + return IndexMetadata.builder(index) + .settings( + settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index 0a223d7b6dc77..97fd2ff52bc13 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -32,19 +32,29 @@ package org.opensearch.action.admin.indices.alias.get; import org.opensearch.Version; +import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.SystemIndices; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; import java.util.Collections; import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportGetAliasesActionTests extends OpenSearchTestCase { private final SystemIndices EMPTY_SYSTEM_INDICES = new SystemIndices(Collections.emptyMap()); @@ -242,6 +252,35 @@ public void testDeprecationWarningEmittedWhenRequestingNonExistingAliasInSystemP ); } + public void testResolveIndices() { + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(systemIndexTestClusterState()); + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.getThreadContext()).thenReturn(threadContext); + + TransportGetAliasesAction action = new TransportGetAliasesAction( + mock(TransportService.class), + clusterService, + threadPool, + mock(ActionFilters.class), + new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + mock(SystemIndices.class) + ); + + { + // Request an alias: Return the indices in the alias + ResolvedIndices resolvedIndices = action.resolveIndices(new GetAliasesRequest("d")); + assertEquals(ResolvedIndices.of("c"), resolvedIndices); + } + + { + // Request an index: Return the index itself + ResolvedIndices resolvedIndices = action.resolveIndices(new GetAliasesRequest().indices("c")); + assertEquals(ResolvedIndices.of("c"), resolvedIndices); + } + } + public ClusterState systemIndexTestClusterState() { return ClusterState.builder(ClusterState.EMPTY_STATE) .metadata( diff --git a/server/src/test/java/org/opensearch/action/admin/indices/create/AutoCreateActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/create/AutoCreateActionTests.java index ff069075c5e34..30fdbe53f4f57 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/create/AutoCreateActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/create/AutoCreateActionTests.java @@ -32,16 +32,30 @@ package org.opensearch.action.admin.indices.create; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.MetadataCreateDataStreamService; +import org.opensearch.cluster.metadata.MetadataCreateIndexService; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; import java.util.Collections; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class AutoCreateActionTests extends OpenSearchTestCase { @@ -83,4 +97,37 @@ public void testResolveAutoCreateDataStreams() { assertThat(result, nullValue()); } + public void testResolveIndices() { + Metadata.Builder mdBuilder = new Metadata.Builder(); + DataStreamTemplate dataStreamTemplate = new DataStreamTemplate(); + mdBuilder.put( + "1", + new ComposableIndexTemplate(Collections.singletonList("datastream-*"), null, null, 20L, null, null, dataStreamTemplate) + ); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + + AutoCreateAction.TransportAction action = new AutoCreateAction.TransportAction( + mock(TransportService.class), + clusterService, + mock(ThreadPool.class), + mock(ActionFilters.class), + new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + mock(MetadataCreateIndexService.class), + mock(MetadataCreateDataStreamService.class) + ); + + ResolvedIndices resolvedIndices = action.resolveIndices(new CreateIndexRequest("")); + assertTrue( + resolvedIndices.toString(), + resolvedIndices.local().containsAny(resolved -> resolved.matches("index-\\d+\\.\\d+\\.\\d+")) + ); + + resolvedIndices = action.resolveIndices(new CreateIndexRequest("datastream-test")); + assertEquals(ResolvedIndices.of("datastream-test"), resolvedIndices); + } + } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamActionTests.java new file mode 100644 index 0000000000000..76f2ec55625b8 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/GetDataStreamActionTests.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.datastream; + +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.DataStream; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.List; + +import static org.opensearch.cluster.DataStreamTestHelper.createBackingIndex; +import static org.opensearch.cluster.DataStreamTestHelper.createTimestampField; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class GetDataStreamActionTests extends OpenSearchTestCase { + public void testResolveIndices() { + String dataStreamName = "datastream-test"; + + IndexMetadata index1 = createBackingIndex(dataStreamName, 1).build(); + IndexMetadata index2 = createBackingIndex(dataStreamName, 2).build(); + + Metadata.Builder mdBuilder = Metadata.builder() + .put(index1, false) + .put(index2, false) + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), List.of(index1.getIndex(), index2.getIndex()), 2)); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + + GetDataStreamAction.TransportAction action = new GetDataStreamAction.TransportAction( + mock(TransportService.class), + clusterService, + mock(ThreadPool.class), + mock(ActionFilters.class), + new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) + ); + + ResolvedIndices resolvedIndices = action.resolveIndices(new GetDataStreamAction.Request(new String[] { "datastream-*" })); + assertEquals(ResolvedIndices.of("datastream-test"), resolvedIndices); + } +} diff --git a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java index 6e0417068ce91..c4c5cca0c3d1a 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java @@ -14,6 +14,8 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.OptionallyResolvedIndices; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; @@ -715,4 +717,88 @@ public void getAllPits(ActionListener getAllPitsListener } } } + + public void testResolveIndices() throws InterruptedException, ExecutionException { + ActionFilters actionFilters = mock(ActionFilters.class); + when(actionFilters.filters()).thenReturn(new ActionFilter[0]); + List knownNodes = new CopyOnWriteArrayList<>(); + try ( + MockTransportService cluster1Transport = startTransport("cluster_1_node", knownNodes, Version.CURRENT); + MockTransportService cluster2Transport = startTransport("cluster_2_node", knownNodes, Version.CURRENT) + ) { + knownNodes.add(cluster1Transport.getLocalDiscoNode()); + knownNodes.add(cluster2Transport.getLocalDiscoNode()); + Collections.shuffle(knownNodes, random()); + + try ( + MockTransportService transportService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { + transportService.start(); + transportService.acceptIncomingRequests(); + SearchTransportService searchTransportService = new SearchTransportService(transportService, null) { + @Override + public Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) { + return new SearchAsyncActionTests.MockConnection(node); + } + }; + PitService pitService = new PitService(clusterServiceMock, searchTransportService, transportService, client); + TransportDeletePitAction action = new TransportDeletePitAction( + transportService, + actionFilters, + namedWriteableRegistry, + pitService + ); + DeletePitRequest deletePITRequest = new DeletePitRequest(pitId); + OptionallyResolvedIndices resolvedIndices = action.resolveIndices(deletePITRequest); + assertEquals(ResolvedIndices.of("idx", "idy"), resolvedIndices); + } + } + } + + public void testResolveIndices_allPits() throws InterruptedException, ExecutionException { + ActionFilters actionFilters = mock(ActionFilters.class); + when(actionFilters.filters()).thenReturn(new ActionFilter[0]); + List knownNodes = new CopyOnWriteArrayList<>(); + try ( + MockTransportService cluster1Transport = startTransport("cluster_1_node", knownNodes, Version.CURRENT); + MockTransportService cluster2Transport = startTransport("cluster_2_node", knownNodes, Version.CURRENT) + ) { + knownNodes.add(cluster1Transport.getLocalDiscoNode()); + knownNodes.add(cluster2Transport.getLocalDiscoNode()); + Collections.shuffle(knownNodes, random()); + + try ( + MockTransportService transportService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { + transportService.start(); + transportService.acceptIncomingRequests(); + SearchTransportService searchTransportService = new SearchTransportService(transportService, null) { + @Override + public Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) { + return new SearchAsyncActionTests.MockConnection(node); + } + }; + PitService pitService = new PitService(clusterServiceMock, searchTransportService, transportService, client); + TransportDeletePitAction action = new TransportDeletePitAction( + transportService, + actionFilters, + namedWriteableRegistry, + pitService + ); + DeletePitRequest deletePITRequest = new DeletePitRequest("_all"); + OptionallyResolvedIndices resolvedIndices = action.resolveIndices(deletePITRequest); + assertEquals(ResolvedIndices.unknown(), resolvedIndices); + } + } + } } diff --git a/server/src/test/java/org/opensearch/action/support/ActionRequestMetadataTests.java b/server/src/test/java/org/opensearch/action/support/ActionRequestMetadataTests.java new file mode 100644 index 0000000000000..b16a2b7a2801d --- /dev/null +++ b/server/src/test/java/org/opensearch/action/support/ActionRequestMetadataTests.java @@ -0,0 +1,85 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.support; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.cluster.metadata.OptionallyResolvedIndices; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.tasks.Task; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Set; + +public class ActionRequestMetadataTests extends OpenSearchTestCase { + public void testResolvedIndices() { + ActionRequestMetadata subject = new ActionRequestMetadata<>( + new TestTransportAction(), + new TestRequest("my_resolution_result") + ); + OptionallyResolvedIndices resolvedIndices = subject.resolvedIndices(); + assertEquals(ResolvedIndices.of("my_resolution_result"), resolvedIndices); + assertSame("ResolvedIndices object is reused on subsequent calls", resolvedIndices, subject.resolvedIndices()); + } + + public void testResolvedIndices_unknown() { + ActionRequestMetadata subject = new ActionRequestMetadata<>( + new TestTransportActionUnsupported(), + new TestRequest("my_resolution_result") + ); + OptionallyResolvedIndices resolvedIndices = subject.resolvedIndices(); + assertFalse(resolvedIndices instanceof ResolvedIndices); + } + + static class TestTransportAction extends TransportAction + implements + TransportIndicesResolvingAction { + + protected TestTransportAction() { + super("action", new ActionFilters(Set.of()), null); + } + + @Override + protected void doExecute(Task task, TestRequest request, ActionListener listener) { + + } + + @Override + public OptionallyResolvedIndices resolveIndices(TestRequest request) { + return ResolvedIndices.of(request.resolutionResult); + } + } + + static class TestRequest extends ActionRequest { + final String resolutionResult; + + TestRequest(String resolutionResult) { + this.resolutionResult = resolutionResult; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + static class TestTransportActionUnsupported extends TransportAction { + + protected TestTransportActionUnsupported() { + super("action", new ActionFilters(Set.of()), null); + } + + @Override + protected void doExecute(Task task, TestRequest request, ActionListener listener) { + + } + } +} diff --git a/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index 4c0023b69488c..cdb490456dcfd 100644 --- a/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -49,6 +49,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.OptionallyResolvedIndices; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; @@ -629,4 +631,10 @@ public void testResultWithTimeouts() throws ExecutionException, InterruptedExcep assertEquals("failed shards", totalFailedShards, response.getFailedShards()); assertEquals("accumulated exceptions", totalFailedShards, response.getShardFailures().length); } + + public void testResolveIndices() { + Request request = new Request(TEST_INDEX); + OptionallyResolvedIndices resolvedIndices = action.resolveIndices(request); + assertEquals(ResolvedIndices.of(TEST_INDEX), resolvedIndices); + } } diff --git a/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java index 118b4e596fc66..5880de71f3910 100644 --- a/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java @@ -43,6 +43,7 @@ import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardIterator; import org.opensearch.cluster.routing.ShardRoutingState; @@ -370,4 +371,21 @@ protected void resolveRequest(ClusterState state, Request request) { } } } + + public void testResolveIndices() { + Request request = new Request().index("test"); + request.shardId = new ShardId("test", "_na_", 0); + setState(clusterService, ClusterStateCreationUtils.state("test", true, ShardRoutingState.STARTED)); + ResolvedIndices resolvedIndices = action.resolveIndices(request); + assertEquals(ResolvedIndices.of("test"), resolvedIndices); + } + + public void testResolveIndices_concreteIndex() { + Request request = new Request().index("test"); + request.concreteIndex("concrete_value"); + request.shardId = new ShardId("test", "_na_", 0); + setState(clusterService, ClusterStateCreationUtils.state("test", true, ShardRoutingState.STARTED)); + ResolvedIndices resolvedIndices = action.resolveIndices(request); + assertEquals(ResolvedIndices.of("concrete_value"), resolvedIndices); + } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java index 8c54b195711af..153e3f389cf05 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -1600,6 +1600,7 @@ public void testResolveExpressions() { indexNameExpressionResolver.resolveExpressions(state, "test-*", "alias-*") ); assertEquals(new HashSet<>(Arrays.asList("test-1", "alias-1")), indexNameExpressionResolver.resolveExpressions(state, "*-1")); + expectThrows(InvalidIndexNameException.class, () -> indexNameExpressionResolver.resolveExpressions(state, "_invalid_index_name")); } public void testFilteringAliases() { @@ -2307,6 +2308,16 @@ public void testDataStreams() { ); assertThat(e.getMessage(), equalTo("no such index [my-data-stream]")); } + { + // Ignore data streams; use different overload + IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; + ResolvedIndices.Local.Concrete concreteResolvedIndices = indexNameExpressionResolver.concreteResolvedIndices( + state, + indicesOptions, + "my-data-stream" + ); + assertThat(concreteResolvedIndices.resolutionErrors().getFirst().getMessage(), equalTo("no such index [my-data-stream]")); + } { // Ignore data streams and allow no indices IndicesOptions indicesOptions = new IndicesOptions( diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java index ad2ec849b291e..7510af7173913 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java @@ -8,13 +8,18 @@ package org.opensearch.cluster.metadata; +import org.opensearch.Version; import org.opensearch.action.OriginalIndices; +import org.opensearch.action.admin.indices.delete.DeleteIndexAction; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; import org.opensearch.core.index.Index; import org.opensearch.index.IndexNotFoundException; import org.opensearch.test.OpenSearchTestCase; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; @@ -27,12 +32,18 @@ public class ResolvedIndicesTests extends OpenSearchTestCase { public void testOfNames() { ResolvedIndices resolvedIndices = ResolvedIndices.of("a", "b", "c"); assertThat(resolvedIndices.local().names(), containsInAnyOrder("a", "b", "c")); + assertThat(resolvedIndices.local().names(ClusterState.EMPTY_STATE), containsInAnyOrder("a", "b", "c")); assertThat(Arrays.asList(resolvedIndices.local().namesAsArray()), containsInAnyOrder("a", "b", "c")); assertFalse(resolvedIndices.local().isEmpty()); assertTrue(resolvedIndices.remote().isEmpty()); assertFalse(resolvedIndices.isEmpty()); } + public void testOfNamesCollection() { + ResolvedIndices resolvedIndices = ResolvedIndices.of(List.of("a", "b")); + assertThat(resolvedIndices.local().names(), containsInAnyOrder("a", "b")); + } + public void testOfConcreteIndices() { ResolvedIndices resolvedIndices = ResolvedIndices.of(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b")); assertThat(resolvedIndices.local().names(), containsInAnyOrder("index_a", "index_b")); @@ -60,6 +71,18 @@ public void testOfEmpty() { assertTrue(resolvedIndices.isEmpty()); } + public void testUnknown() { + OptionallyResolvedIndices resolvedIndices = ResolvedIndices.unknown(); + assertFalse(resolvedIndices instanceof ResolvedIndices); + resolvedIndices = OptionallyResolvedIndices.unknown(); + assertFalse(resolvedIndices instanceof ResolvedIndices); + assertFalse(resolvedIndices.local().isEmpty()); + assertTrue(resolvedIndices.local().contains("whatever")); + assertTrue(resolvedIndices.local().containsAny(List.of("whatever"))); + assertTrue(resolvedIndices.local().containsAny(i -> false)); + + } + public void testWithRemoteIndices() { Map remoteIndices = Map.of( "remote_cluster", @@ -74,6 +97,12 @@ public void testWithRemoteIndices() { assertThat(resolvedIndices.remote().asRawExpressions(), containsInAnyOrder("remote_cluster:remote_index")); } + public void testWithoutRemoteIndices() { + ResolvedIndices resolvedIndices = ResolvedIndices.of(new String[0]); + assertTrue(resolvedIndices.remote().asClusterToOriginalIndicesMap().isEmpty()); + assertTrue(resolvedIndices.remote().asRawExpressions().isEmpty()); + } + public void testLocalConcreteOf() { Set indices = Set.of(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b")); Set names = Set.of("index_a", "index_b", "index_c"); @@ -96,4 +125,46 @@ public void testLocalConcreteWithResolutionErrors() { assertThat(concrete.concreteIndices(), containsInAnyOrder(new Index("index_a", "uuid_a"), new Index("index_b", "uuid_b"))); } + public void testWithLocalSubActions() { + ResolvedIndices resolvedIndices = ResolvedIndices.of("a", "b", "c") + .withLocalSubActions(DeleteIndexAction.INSTANCE, ResolvedIndices.Local.of("x", "y", "z")); + assertThat(resolvedIndices.local().names(), containsInAnyOrder("a", "b", "c")); + assertThat(resolvedIndices.local().subActions().get(DeleteIndexAction.INSTANCE).names(), containsInAnyOrder("x", "y", "z")); + } + + public void testLocalOf() { + ResolvedIndices.Local local = ResolvedIndices.Local.of(List.of("o", "p", "q")); + assertThat(local.names(), containsInAnyOrder("o", "p", "q")); + } + + public void testNamesOfIndices() { + Metadata.Builder metadata = Metadata.builder() + .put(indexBuilder("index_a1").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_a2").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_b1")); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); + ResolvedIndices resolvedIndices = ResolvedIndices.of("index_not_existing", "alias_a", "index_b1"); + assertThat( + resolvedIndices.local().namesOfIndices(clusterState), + containsInAnyOrder("index_not_existing", "index_a1", "index_a2", "index_b1") + ); + } + + public void testContains() { + ResolvedIndices resolvedIndices = ResolvedIndices.of("a", "b", "c"); + assertTrue(resolvedIndices.local().contains("a")); + assertFalse(resolvedIndices.local().contains("x")); + assertTrue(resolvedIndices.local().containsAny(List.of("a", "x"))); + assertFalse(resolvedIndices.local().containsAny(List.of("x", "y"))); + assertTrue(resolvedIndices.local().containsAny(i -> i.equals("a"))); + assertFalse(resolvedIndices.local().containsAny(i -> i.equals("z"))); + + } + + private static IndexMetadata.Builder indexBuilder(String index) { + return IndexMetadata.builder(index) + .settings( + settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ); + } } diff --git a/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java b/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java index ca9370645dec3..5cfd2c06d97e4 100644 --- a/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java +++ b/server/src/test/java/org/opensearch/indices/SystemIndicesTests.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import static java.util.Collections.emptyMap; @@ -189,6 +190,9 @@ public void testSystemIndexMatching() { equalTo(Set.of(".system-index1", ".system-index-pattern1")) ); assertThat(SystemIndexRegistry.matchesSystemIndexPattern(Set.of(".not-system")), equalTo(Collections.emptySet())); + + assertTrue(SystemIndexRegistry.matchesSystemIndexPattern(".system-index1")); + assertFalse(SystemIndexRegistry.matchesSystemIndexPattern(".not-system-index")); } public void testRegisteredSystemIndexGetAllDescriptors() { @@ -259,6 +263,18 @@ public void testRegisteredSystemIndexMatchingForPlugin() { Set.of("other-index") ); assertEquals(0, noMatchingSystemIndices.size()); + + Predicate plugin1systemIndexPredicate = SystemIndexRegistry.getPluginSystemIndexPredicate( + SystemIndexPlugin1.class.getCanonicalName() + ); + assertTrue(plugin1systemIndexPredicate.test(SystemIndexPlugin1.SYSTEM_INDEX_1)); + assertFalse(plugin1systemIndexPredicate.test(SystemIndexPlugin2.SYSTEM_INDEX_2)); + assertFalse(plugin1systemIndexPredicate.test("unknown")); + + Predicate unknownPluginSystemIndexPredicate = SystemIndexRegistry.getPluginSystemIndexPredicate("unknown_plugin"); + assertFalse(unknownPluginSystemIndexPredicate.test(SystemIndexPlugin1.SYSTEM_INDEX_1)); + assertFalse(unknownPluginSystemIndexPredicate.test(SystemIndexPlugin2.SYSTEM_INDEX_2)); + assertFalse(unknownPluginSystemIndexPredicate.test("unknown")); } static final class SystemIndexPlugin1 extends Plugin implements SystemIndexPlugin { From db803e6d2345424d2c2f9174c0d9437485124334 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 3 Sep 2025 09:54:24 +0200 Subject: [PATCH 10/24] Test fix Signed-off-by: Nils Bandener --- .../action/search/TransportDeletePitActionTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java index c4c5cca0c3d1a..0a31d2ad0b352 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java @@ -719,6 +719,8 @@ public void getAllPits(ActionListener getAllPitsListener } public void testResolveIndices() throws InterruptedException, ExecutionException { + NodeClient client = new NodeClient(settings, threadPool); + client.initialize(null, null, null, namedWriteableRegistry); ActionFilters actionFilters = mock(ActionFilters.class); when(actionFilters.filters()).thenReturn(new ActionFilter[0]); List knownNodes = new CopyOnWriteArrayList<>(); @@ -761,6 +763,8 @@ public Transport.Connection getConnection(String clusterAlias, DiscoveryNode nod } public void testResolveIndices_allPits() throws InterruptedException, ExecutionException { + NodeClient client = new NodeClient(settings, threadPool); + client.initialize(null, null, null, namedWriteableRegistry); ActionFilters actionFilters = mock(ActionFilters.class); when(actionFilters.filters()).thenReturn(new ActionFilter[0]); List knownNodes = new CopyOnWriteArrayList<>(); From 55f92b63a3213838c7e08ed9259f7a193de1554d Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 5 Sep 2025 07:53:10 +0200 Subject: [PATCH 11/24] Additional tests Signed-off-by: Nils Bandener --- .../rollover/TransportRolloverAction.java | 11 +- .../indices/shrink/TransportResizeAction.java | 7 +- .../cluster/metadata/ResolvedIndices.java | 6 +- .../put/TransportPutMappingActionTests.java | 57 ++++++++- .../TransportRolloverActionTests.java | 62 +++++++++ .../TransportPitSegmentsActionTests.java | 75 +++++++++++ .../shrink/TransportResizeActionTests.java | 35 +++++ .../search/TransportSearchActionTests.java | 120 ++++++++++++++++++ .../metadata/ResolvedIndicesTests.java | 38 +++++- 9 files changed, 399 insertions(+), 12 deletions(-) create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsActionTests.java diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java index 8168c9aab98df..876cf84c88e5a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -33,6 +33,7 @@ package org.opensearch.action.admin.indices.rollover; import org.opensearch.OpenSearchException; +import org.opensearch.action.admin.indices.create.CreateIndexAction; import org.opensearch.action.admin.indices.stats.IndicesStatsAction; import org.opensearch.action.admin.indices.stats.IndicesStatsRequest; import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; @@ -277,13 +278,19 @@ public ResolvedIndices resolveIndices(RolloverRequest rolloverRequest) { true ); - return ResolvedIndices.of(preResult.sourceIndexName, preResult.rolloverIndexName); + return ResolvedIndices.of(rolloverRequest.getRolloverTarget()) + .withLocalSubActions(CreateIndexAction.INSTANCE, ResolvedIndices.Local.of(preResult.rolloverIndexName)); } catch (Exception e) { // Exceptions are mostly occurring due to validation errors (e.g. non-existing indices). // These are not propagated to the caller because it should be still // the clusterManagerOperation() that should report these failures. // Instead, we return a basic result which still allows privilege evaluation. - return ResolvedIndices.ofNonNull(rolloverRequest.getNewIndexName(), rolloverRequest.getRolloverTarget()); + if (rolloverRequest.getNewIndexName() != null) { + return ResolvedIndices.of(rolloverRequest.getRolloverTarget()) + .withLocalSubActions(CreateIndexAction.INSTANCE, ResolvedIndices.Local.of(rolloverRequest.getNewIndexName())); + } else { + return ResolvedIndices.of(rolloverRequest.getRolloverTarget()); + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java index 7bf6d6b023eab..bd77fb967fd43 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java @@ -33,6 +33,7 @@ package org.opensearch.action.admin.indices.shrink; import org.apache.lucene.index.IndexWriter; +import org.opensearch.action.admin.indices.create.CreateIndexAction; import org.opensearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.stats.IndexShardStats; @@ -231,9 +232,13 @@ protected void clusterManagerOperation( } + /** + * Reports the resize source index as the main resolved index. The target index is reported as sub-action "indices:admin/create". + */ @Override public ResolvedIndices resolveIndices(ResizeRequest resizeRequest) { - return ResolvedIndices.of(resolveSourceIndex(resizeRequest), resolveTargetIndex(resizeRequest)); + return ResolvedIndices.of(resolveSourceIndex(resizeRequest)) + .withLocalSubActions(CreateIndexAction.INSTANCE, ResolvedIndices.Local.of(resolveTargetIndex(resizeRequest))); } // static for unittesting this method diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java index 51db46c94e3fd..ca954e339ae6c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java @@ -149,7 +149,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - return this.local.hashCode() + this.remote.hashCode(); + return this.local.hashCode() + this.remote.hashCode() * 31; } /** @@ -336,7 +336,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - return this.names.hashCode() + this.subActions.hashCode(); + return this.names.hashCode() + this.subActions.hashCode() * 31; } /** @@ -467,7 +467,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - return super.hashCode() + this.concreteIndices.hashCode() * 11; + return super.hashCode() * 31 + this.concreteIndices.hashCode(); } List resolutionErrors() { diff --git a/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingActionTests.java index dc9bd6b0bdb9d..77d7559e5534f 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingActionTests.java @@ -8,15 +8,24 @@ package org.opensearch.action.admin.indices.mapping.put; +import org.opensearch.Version; import org.opensearch.action.RequestValidators; import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; +import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.MetadataMappingService; import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.mapper.MappingTransformerRegistry; import org.opensearch.test.OpenSearchTestCase; @@ -31,6 +40,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -42,8 +52,6 @@ public class TransportPutMappingActionTests extends OpenSearchTestCase { private ActionFilters actionFilters; @Mock private ThreadPool threadPool; - @Mock - private IndexNameExpressionResolver indexNameExpressionResolver; @Mock private MetadataMappingService metadataMappingService; @@ -54,7 +62,6 @@ public class TransportPutMappingActionTests extends OpenSearchTestCase { @Mock private RequestValidators requestValidators; - @Mock private ClusterState clusterState; @Mock @@ -66,12 +73,31 @@ public class TransportPutMappingActionTests extends OpenSearchTestCase { public void setup() { MockitoAnnotations.openMocks(this); + IndexMetadata.Builder index = IndexMetadata.builder("index") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + IndexMetadata.Builder indexA1 = IndexMetadata.builder("index_a1") + .putAlias(AliasMetadata.builder("alias_a").build()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + IndexMetadata.Builder indexA2 = IndexMetadata.builder("index_a2") + .putAlias(AliasMetadata.builder("alias_a").writeIndex(true).build()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(index).put(indexA1).put(indexA2)).build(); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)); + ActionFilter[] emptyActionFilters = new ActionFilter[] {}; - when(indexNameExpressionResolver.concreteResolvedIndices(any(), any())).thenReturn(ResolvedIndices.Local.Concrete.empty()); when(actionFilters.filters()).thenReturn(emptyActionFilters); action = new TransportPutMappingAction( transportService, - null, // ClusterService not needed for this test + clusterService, threadPool, metadataMappingService, actionFilters, @@ -109,4 +135,25 @@ public void testClusterManagerOperation_transformedMappingUsed() { PutMappingClusterStateUpdateRequest capturedRequest = updateRequestCaptor.getValue(); assertEquals(transformedMapping, capturedRequest.source()); } + + public void testResolveIndices() { + { + ResolvedIndices resolvedIndices = action.resolveIndices(new PutMappingRequest("index_a*")); + assertEquals(ResolvedIndices.of("index_a1", "index_a2"), resolvedIndices); + } + { + ResolvedIndices resolvedIndices = action.resolveIndices(new PutMappingRequest("alias_a")); + assertEquals(ResolvedIndices.of("alias_a"), resolvedIndices); + } + { + ResolvedIndices resolvedIndices = action.resolveIndices(new PutMappingRequest("alias_a").writeIndexOnly(true)); + assertEquals(ResolvedIndices.of("index_a2"), resolvedIndices); + } + { + ResolvedIndices resolvedIndices = action.resolveIndices( + new PutMappingRequest().setConcreteIndex(new Index("index_c", "index_c_uuid")) + ); + assertEquals(ResolvedIndices.of("index_c"), resolvedIndices); + } + } } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 6cef1049b3b50..6ff7c3b722056 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -34,6 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionRequest; +import org.opensearch.action.admin.indices.create.CreateIndexAction; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndexStats; import org.opensearch.action.admin.indices.stats.IndicesStatsAction; @@ -50,6 +51,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.MetadataCreateIndexService; import org.opensearch.cluster.metadata.MetadataIndexAliasesService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.cluster.routing.ShardRouting; @@ -58,6 +60,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.util.set.Sets; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.unit.ByteSizeUnit; @@ -326,6 +329,65 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr assertThat(response.getConditionStatus().get("[max_docs: 300]"), is(true)); } + public void testResolveIndices() { + IndexMetadata.Builder indexMetadata = IndexMetadata.builder("logs-index-000001") + .putAlias(AliasMetadata.builder("logs-alias").writeIndex(false).build()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + IndexMetadata.Builder indexMetadata2 = IndexMetadata.builder("logs-index-000002") + .putAlias(AliasMetadata.builder("logs-alias").writeIndex(true).build()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + ClusterState stateBefore = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().put(indexMetadata).put(indexMetadata2)) + .build(); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(stateBefore); + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.getThreadContext()).thenReturn(threadContext); + MetadataCreateIndexService createIndexService = mock(MetadataCreateIndexService.class); + MetadataIndexAliasesService metadataIndexAliasesService = mock(MetadataIndexAliasesService.class); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)); + MetadataRolloverService metadataRolloverService = new MetadataRolloverService( + threadPool, + createIndexService, + metadataIndexAliasesService, + indexNameExpressionResolver + ); + + TransportRolloverAction action = new TransportRolloverAction( + mock(TransportService.class), + clusterService, + threadPool, + mock(ActionFilters.class), + indexNameExpressionResolver, + metadataRolloverService, + mock(Client.class) + ); + + { + ResolvedIndices resolvedIndices = action.resolveIndices(new RolloverRequest("logs-alias", null)); + assertEquals( + ResolvedIndices.of("logs-alias") + .withLocalSubActions(CreateIndexAction.INSTANCE, ResolvedIndices.Local.of("logs-index-000003")), + resolvedIndices + ); + } + + { + ResolvedIndices resolvedIndices = action.resolveIndices(new RolloverRequest("logs-alias", "explicit-index")); + assertEquals( + ResolvedIndices.of("logs-alias") + .withLocalSubActions(CreateIndexAction.INSTANCE, ResolvedIndices.Local.of("explicit-index")), + resolvedIndices + ); + } + } + private IndicesStatsResponse createIndicesStatResponse(String indexName, long totalDocs, long primariesDocs) { final CommonStats primaryStats = mock(CommonStats.class); when(primaryStats.getDocs()).thenReturn(new DocsStats(primariesDocs, 0, between(1, 10000))); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsActionTests.java new file mode 100644 index 0000000000000..0b3686505d4d8 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/segments/TransportPitSegmentsActionTests.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.segments; + +import org.opensearch.action.search.PitService; +import org.opensearch.action.search.PitTestsUtil; +import org.opensearch.action.search.SearchTransportService; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.OptionallyResolvedIndices; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.index.query.IdsQueryBuilder; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.indices.IndicesService; +import org.opensearch.search.SearchService; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.TransportService; +import org.opensearch.transport.client.node.NodeClient; + +import java.util.Arrays; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportPitSegmentsActionTests extends OpenSearchTestCase { + public void testResolveIndices() { + ClusterService clusterService = mock(ClusterService.class); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)); + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry( + Arrays.asList( + new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, IdsQueryBuilder.NAME, IdsQueryBuilder::new) + ) + ); + NodeClient client = mock(NodeClient.class); + when(client.getNamedWriteableRegistry()).thenReturn(namedWriteableRegistry); + PitService pitService = new PitService(clusterService, mock(SearchTransportService.class), mock(TransportService.class), client); + + TransportPitSegmentsAction action = new TransportPitSegmentsAction( + clusterService, + mock(TransportService.class), + mock(IndicesService.class), + mock(ActionFilters.class), + indexNameExpressionResolver, + mock(SearchService.class), + namedWriteableRegistry, + pitService + ); + + { + // The generated pitId will contain the indices "idx" and "idy" + String pitId = PitTestsUtil.getPitId(); + OptionallyResolvedIndices resolvedIndices = action.resolveIndices(new PitSegmentsRequest(pitId)); + assertEquals(ResolvedIndices.of("idx", "idy"), resolvedIndices); + } + + { + OptionallyResolvedIndices resolvedIndices = action.resolveIndices(new PitSegmentsRequest("_all")); + assertFalse("result is unknown", resolvedIndices instanceof ResolvedIndices); + } + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java index 18069ede796af..6cf9aa4bc15f7 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -34,7 +34,9 @@ import org.apache.lucene.index.IndexWriter; import org.opensearch.Version; +import org.opensearch.action.admin.indices.create.CreateIndexAction; import org.opensearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -42,7 +44,10 @@ import org.opensearch.cluster.OpenSearchAllocationTestCase; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.MetadataCreateIndexService; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; @@ -51,8 +56,10 @@ import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.shard.DocsStats; import org.opensearch.index.store.StoreStats; @@ -60,6 +67,9 @@ import org.opensearch.snapshots.EmptySnapshotsInfoService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.gateway.TestGatewayAllocator; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.opensearch.transport.client.Client; import java.util.Arrays; import java.util.Collections; @@ -73,6 +83,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.hamcrest.CoreMatchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportResizeActionTests extends OpenSearchTestCase { @@ -721,6 +733,29 @@ public void testResizeFailuresDuringMigration() { } } + public void testResolveIndices() { + ClusterService clusterService = mock(ClusterService.class); + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.getThreadContext()).thenReturn(threadContext); + + TransportResizeAction action = new TransportResizeAction( + mock(TransportService.class), + clusterService, + threadPool, + mock(MetadataCreateIndexService.class), + mock(ActionFilters.class), + new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + mock(Client.class) + ); + + ResolvedIndices resolvedIndices = action.resolveIndices(new ResizeRequest("target-index", "source-index")); + assertEquals( + ResolvedIndices.of("source-index").withLocalSubActions(CreateIndexAction.INSTANCE, ResolvedIndices.Local.of("target-index")), + resolvedIndices + ); + } + private DiscoveryNode newNode(String nodeId) { final Set roles = Collections.unmodifiableSet( new HashSet<>(Arrays.asList(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE)) diff --git a/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java index 0a0015ae8cbf6..4cde4a2fb49e8 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportSearchActionTests.java @@ -40,42 +40,60 @@ import org.opensearch.action.OriginalIndicesTests; import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsGroup; import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsResponse; +import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.IndicesOptions; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.GroupShardsIterator; import org.opensearch.cluster.routing.GroupShardsIteratorTests; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.query.IdsQueryBuilder; import org.opensearch.index.query.InnerHitBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.query.TermsQueryBuilder; import org.opensearch.search.Scroll; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.search.SearchService; import org.opensearch.search.SearchShardTarget; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.collapse.CollapseBuilder; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.pipeline.SearchPipelineService; import org.opensearch.search.sort.SortBuilders; +import org.opensearch.tasks.TaskResourceTrackingService; +import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; @@ -92,6 +110,7 @@ import org.opensearch.transport.TransportRequest; import org.opensearch.transport.TransportRequestOptions; import org.opensearch.transport.TransportService; +import org.opensearch.transport.client.node.NodeClient; import java.util.ArrayList; import java.util.Arrays; @@ -114,6 +133,8 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.startsWith; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportSearchActionTests extends OpenSearchTestCase { @@ -1166,4 +1187,103 @@ public void testShouldPreFilterSearchShardsWithReadOnly() { ); } } + + public void testResolveIndices() { + IndexMetadata.Builder indexA1 = IndexMetadata.builder("index_a1") + .putAlias(AliasMetadata.builder("alias_a").build()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + IndexMetadata.Builder indexA2 = IndexMetadata.builder("index_a2") + .putAlias(AliasMetadata.builder("alias_a").build()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().put(indexA1).put(indexA2)) + .build(); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)); + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry( + Arrays.asList( + new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new), + new NamedWriteableRegistry.Entry(QueryBuilder.class, IdsQueryBuilder.NAME, IdsQueryBuilder::new) + ) + ); + int numClusters = 2; + DiscoveryNode[] nodes = new DiscoveryNode[numClusters]; + Map remoteIndicesByCluster = new HashMap<>(); + Settings.Builder builder = Settings.builder(); + MockTransportService[] mockTransportServices = startTransport(numClusters, nodes, remoteIndicesByCluster, builder); + Settings settings = builder.build(); + try ( + MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, NoopTracer.INSTANCE) + ) { + service.start(); + service.acceptIncomingRequests(); + RemoteClusterService remoteClusterService = service.getRemoteClusterService(); + + SearchTransportService searchTransportService = mock(SearchTransportService.class); + when(searchTransportService.getRemoteClusterService()).thenReturn(remoteClusterService); + + TransportSearchAction action = new TransportSearchAction( + mock(NodeClient.class), + threadPool, + mock(CircuitBreakerService.class), + mock(TransportService.class), + mock(SearchService.class), + searchTransportService, + new SearchPhaseController(namedWriteableRegistry, (searchSourceBuilder) -> null), + clusterService, + mock(ActionFilters.class), + indexNameExpressionResolver, + namedWriteableRegistry, + mock(SearchPipelineService.class), + mock(MetricsRegistry.class), + new SearchRequestOperationsCompositeListenerFactory(), + mock(Tracer.class), + mock(TaskResourceTrackingService.class) + ); + + // Actual test cases start here: + + { + ResolvedIndices resolvedIndices = action.resolveIndices(new SearchRequest("index_*")); + assertEquals(ResolvedIndices.of("index_a1", "index_a2"), resolvedIndices); + } + + { + ResolvedIndices resolvedIndices = action.resolveIndices(new SearchRequest("alias_a")); + assertEquals(ResolvedIndices.of("alias_a"), resolvedIndices); + } + + { + // The generated pitId will contain the indices "idx" and "idy" + String pitId = PitTestsUtil.getPitId(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(pitId)); + ResolvedIndices resolvedIndices = action.resolveIndices(new SearchRequest().source(searchSourceBuilder)); + assertEquals(ResolvedIndices.of("idx", "idy"), resolvedIndices); + } + + { + ResolvedIndices resolvedIndices = action.resolveIndices(new SearchRequest("index_*", "remote1:remote_index")); + assertEquals( + ResolvedIndices.of("index_a1", "index_a2") + .withRemoteIndices( + Map.of("remote1", new OriginalIndices(new String[] { "remote_index" }, IndicesOptions.LENIENT_EXPAND_OPEN)) + ), + resolvedIndices + ); + } + + } finally { + for (MockTransportService mockTransportService : mockTransportServices) { + mockTransportService.close(); + } + } + } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java index 7510af7173913..d406bf3c582f3 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java @@ -80,7 +80,6 @@ public void testUnknown() { assertTrue(resolvedIndices.local().contains("whatever")); assertTrue(resolvedIndices.local().containsAny(List.of("whatever"))); assertTrue(resolvedIndices.local().containsAny(i -> false)); - } public void testWithRemoteIndices() { @@ -95,6 +94,7 @@ public void testWithRemoteIndices() { resolvedIndices.remote().asClusterToOriginalIndicesMap().get("remote_cluster").indices() ); assertThat(resolvedIndices.remote().asRawExpressions(), containsInAnyOrder("remote_cluster:remote_index")); + assertEquals(resolvedIndices.remote().asRawExpressions(), Arrays.asList(resolvedIndices.remote().asRawExpressionsArray())); } public void testWithoutRemoteIndices() { @@ -158,7 +158,43 @@ public void testContains() { assertFalse(resolvedIndices.local().containsAny(List.of("x", "y"))); assertTrue(resolvedIndices.local().containsAny(i -> i.equals("a"))); assertFalse(resolvedIndices.local().containsAny(i -> i.equals("z"))); + } + public void testEqualsHashCode() { + ResolvedIndices resolvedIndices1 = ResolvedIndices.of("index") + .withRemoteIndices(Map.of("remote", new OriginalIndices(new String[] { "remote_index" }, IndicesOptions.LENIENT_EXPAND_OPEN))) + .withLocalSubActions(DeleteIndexAction.INSTANCE, ResolvedIndices.Local.of("other")); + ResolvedIndices resolvedIndices2 = ResolvedIndices.of("index") + .withRemoteIndices(Map.of("remote", new OriginalIndices(new String[] { "remote_index" }, IndicesOptions.LENIENT_EXPAND_OPEN))) + .withLocalSubActions(DeleteIndexAction.INSTANCE, ResolvedIndices.Local.of("other")); + assertEquals(resolvedIndices1.hashCode(), resolvedIndices2.hashCode()); + assertTrue(resolvedIndices1 + " equals " + resolvedIndices2, resolvedIndices1.equals(resolvedIndices2)); + assertTrue(resolvedIndices2 + " equals " + resolvedIndices1, resolvedIndices2.equals(resolvedIndices1)); + ResolvedIndices resolvedIndices3 = resolvedIndices2.withRemoteIndices( + Map.of("remote2", new OriginalIndices(new String[] { "remote_index" }, IndicesOptions.LENIENT_EXPAND_OPEN)) + ); + assertNotEquals(resolvedIndices1.hashCode(), resolvedIndices3.hashCode()); + assertFalse(resolvedIndices1.equals(resolvedIndices3)); + assertFalse(resolvedIndices3.equals(resolvedIndices1)); + } + + public void testHashCodeForConcreteIndices() { + ResolvedIndices resolvedIndices1 = ResolvedIndices.of(ResolvedIndices.Local.Concrete.of(new Index("index", "index_uuid"))) + .withRemoteIndices(Map.of("remote", new OriginalIndices(new String[] { "remote_index" }, IndicesOptions.LENIENT_EXPAND_OPEN))) + .withLocalSubActions(DeleteIndexAction.INSTANCE, ResolvedIndices.Local.of("other")); + ResolvedIndices resolvedIndices2 = ResolvedIndices.of(ResolvedIndices.Local.Concrete.of(new Index("index", "index_uuid"))) + .withRemoteIndices(Map.of("remote", new OriginalIndices(new String[] { "remote_index" }, IndicesOptions.LENIENT_EXPAND_OPEN))) + .withLocalSubActions(DeleteIndexAction.INSTANCE, ResolvedIndices.Local.of("other")); + assertEquals(resolvedIndices1.hashCode(), resolvedIndices2.hashCode()); + assertTrue(resolvedIndices1 + " equals " + resolvedIndices2, resolvedIndices1.equals(resolvedIndices2)); + assertTrue(resolvedIndices2 + " equals " + resolvedIndices1, resolvedIndices2.equals(resolvedIndices1)); + ResolvedIndices resolvedIndices3 = resolvedIndices2.withLocalSubActions( + DeleteIndexAction.INSTANCE, + ResolvedIndices.Local.of("other2") + ); + assertNotEquals(resolvedIndices1.hashCode(), resolvedIndices3.hashCode()); + assertFalse(resolvedIndices1.equals(resolvedIndices3)); + assertFalse(resolvedIndices3.equals(resolvedIndices1)); } private static IndexMetadata.Builder indexBuilder(String index) { From 16e0096464b98d474288da9b0b670a626090c213 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 5 Sep 2025 14:14:41 +0200 Subject: [PATCH 12/24] API simplifications Signed-off-by: Nils Bandener --- .../cluster/metadata/ResolvedIndices.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java index ca954e339ae6c..5492410ec7041 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java @@ -159,7 +159,7 @@ public int hashCode() { public static class Local extends OptionallyResolvedIndices.Local { protected final Set names; protected final OriginalIndices originalIndices; - protected final Map, Local> subActions; + protected final Map subActions; private Set namesOfIndices; public static Local of(Collection names) { @@ -177,7 +177,7 @@ public static Local of(String... names) { * For this reason, this constructor is private. This contract is guaranteed by the static * constructor methods in this file. */ - private Local(Set names, OriginalIndices originalIndices, Map, Local> subActions) { + private Local(Set names, OriginalIndices originalIndices, Map subActions) { this.names = names; this.originalIndices = originalIndices; this.subActions = subActions; @@ -266,7 +266,7 @@ public OriginalIndices originalIndices() { * For example, the swiss-army-knife IndicesAliases action can delete indices. The subActions() property * can be used to specify indices with such special roles. */ - public Map, Local> subActions() { + public Map subActions() { return this.subActions; } @@ -310,12 +310,16 @@ public ResolvedIndices.Local withOriginalIndices(OriginalIndices originalIndices return new Local(this.names, originalIndices, this.subActions); } - public ResolvedIndices.Local withSubActions(ActionType actionType, ResolvedIndices.Local local) { - Map, Local> subActions = new HashMap<>(this.subActions); - subActions.put(actionType, local); + public ResolvedIndices.Local withSubActions(String key, ResolvedIndices.Local local) { + Map subActions = new HashMap<>(this.subActions); + subActions.put(key, local); return new Local(this.names, this.originalIndices, Collections.unmodifiableMap(subActions)); } + public ResolvedIndices.Local withSubActions(ActionType actionType, ResolvedIndices.Local local) { + return this.withSubActions(actionType.name(), local); + } + @Override public String toString() { if (this.subActions.isEmpty()) { @@ -384,7 +388,7 @@ private Concrete( Set concreteIndices, Set names, OriginalIndices originalIndices, - Map, Local> subActions, + Map subActions, List resolutionErrors ) { super(names, originalIndices, subActions); @@ -420,12 +424,14 @@ public String[] namesOfConcreteIndicesAsArray() { return this.concreteIndices().stream().map(Index::getName).toArray(String[]::new); } + @Override public ResolvedIndices.Local.Concrete withOriginalIndices(OriginalIndices originalIndices) { return new Concrete(this.concreteIndices, this.names, originalIndices, this.subActions, resolutionErrors); } - public ResolvedIndices.Local withSubActions(ActionType actionType, ResolvedIndices.Local local) { - Map, Local> subActions = new HashMap<>(this.subActions); + @Override + public ResolvedIndices.Local withSubActions(String actionType, ResolvedIndices.Local local) { + Map subActions = new HashMap<>(this.subActions); subActions.put(actionType, local); return new Concrete(this.concreteIndices, this.names, this.originalIndices, subActions, resolutionErrors); } From 19a54802060189769ecff72ca961f9d7f63cb3ce Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 5 Sep 2025 15:01:00 +0200 Subject: [PATCH 13/24] Test fix Signed-off-by: Nils Bandener --- .../org/opensearch/cluster/metadata/ResolvedIndicesTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java index d406bf3c582f3..e129cbf1fa02b 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java @@ -129,7 +129,7 @@ public void testWithLocalSubActions() { ResolvedIndices resolvedIndices = ResolvedIndices.of("a", "b", "c") .withLocalSubActions(DeleteIndexAction.INSTANCE, ResolvedIndices.Local.of("x", "y", "z")); assertThat(resolvedIndices.local().names(), containsInAnyOrder("a", "b", "c")); - assertThat(resolvedIndices.local().subActions().get(DeleteIndexAction.INSTANCE).names(), containsInAnyOrder("x", "y", "z")); + assertThat(resolvedIndices.local().subActions().get(DeleteIndexAction.NAME).names(), containsInAnyOrder("x", "y", "z")); } public void testLocalOf() { From 140e0020de637e718a47fe80ffa9e99497ca1d7d Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Sun, 7 Sep 2025 07:28:34 +0200 Subject: [PATCH 14/24] Added resolveIndices() for composite index actions Signed-off-by: Nils Bandener --- .../index/reindex/TransportReindexAction.java | 24 +++++++++++++++++-- .../action/get/TransportMultiGetAction.java | 12 +++++++++- .../TransportMultiTermVectorsAction.java | 13 +++++++++- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportReindexAction.java index a4998ebc0b652..7ca8421674b43 100644 --- a/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/opensearch/index/reindex/TransportReindexAction.java @@ -32,10 +32,14 @@ package org.opensearch.index.reindex; +import org.opensearch.action.index.IndexAction; +import org.opensearch.action.search.TransportSearchAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.AutoCreateIndex; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Setting; @@ -56,7 +60,9 @@ import static java.util.Collections.emptyList; -public class TransportReindexAction extends HandledTransportAction { +public class TransportReindexAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { public static final Setting> REMOTE_CLUSTER_ALLOWLIST = Setting.listSetting( "reindex.remote.allowlist", emptyList(), @@ -88,6 +94,8 @@ public class TransportReindexAction extends HandledTransportAction { +public class TransportMultiGetAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final ClusterService clusterService; private final TransportShardMultiGetAction shardAction; @@ -174,4 +179,9 @@ private void finishHim() { private static MultiGetItemResponse newItemFailure(String index, String id, Exception exception) { return new MultiGetItemResponse(null, new MultiGetResponse.Failure(index, id, exception)); } + + @Override + public ResolvedIndices resolveIndices(MultiGetRequest request) { + return ResolvedIndices.of(request.items.stream().map(MultiGetRequest.Item::index).collect(Collectors.toSet())); + } } diff --git a/server/src/main/java/org/opensearch/action/termvectors/TransportMultiTermVectorsAction.java b/server/src/main/java/org/opensearch/action/termvectors/TransportMultiTermVectorsAction.java index 0364f36106cb0..2348c669cddfc 100644 --- a/server/src/main/java/org/opensearch/action/termvectors/TransportMultiTermVectorsAction.java +++ b/server/src/main/java/org/opensearch/action/termvectors/TransportMultiTermVectorsAction.java @@ -35,9 +35,12 @@ import org.opensearch.action.RoutingMissingException; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.action.support.TransportIndicesResolvingAction; +import org.opensearch.action.support.single.shard.SingleShardRequest; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.util.concurrent.AtomicArray; @@ -50,13 +53,16 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; /** * Performs the multi term get operation. * * @opensearch.internal */ -public class TransportMultiTermVectorsAction extends HandledTransportAction { +public class TransportMultiTermVectorsAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final ClusterService clusterService; private final TransportShardMultiTermsVectorAction shardAction; @@ -186,4 +192,9 @@ private void finishHim() { }); } } + + @Override + public ResolvedIndices resolveIndices(MultiTermVectorsRequest request) { + return ResolvedIndices.of(request.getRequests().stream().map(SingleShardRequest::index).collect(Collectors.toSet())); + } } From d602cc70cc8dc1b9ec5881b0ad85061f13fc30ba Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 9 Sep 2025 09:23:49 +0200 Subject: [PATCH 15/24] Improvements Signed-off-by: Nils Bandener --- .../TransportSearchTemplateAction.java | 10 +- .../reindex/TransportReindexActionTests.java | 64 ++++++++ .../alias/get/TransportGetAliasesAction.java | 19 ++- .../create/TransportCreateIndexAction.java | 14 +- .../shard/TransportSingleShardAction.java | 2 +- .../cluster/metadata/ResolvedIndices.java | 153 ++++++++++++++++-- .../get/TransportGetAliasesActionTests.java | 10 +- ...ortInstanceSingleOperationActionTests.java | 8 + .../metadata/ResolvedIndicesTests.java | 14 ++ 9 files changed, 273 insertions(+), 21 deletions(-) create mode 100644 modules/reindex/src/test/java/org/opensearch/index/reindex/TransportReindexActionTests.java diff --git a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java index 85cf7a606a08f..62c2ef35f5c94 100644 --- a/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/opensearch/script/mustache/TransportSearchTemplateAction.java @@ -38,6 +38,7 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.TransportIndicesResolvingAction; +import org.opensearch.cluster.metadata.OptionallyResolvedIndices; import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.common.inject.Inject; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -191,7 +192,12 @@ private static void checkRestTotalHitsAsInt(SearchRequest searchRequest, SearchS } @Override - public ResolvedIndices resolveIndices(SearchTemplateRequest request) { - return transportSearchAction.resolveIndices(request.getRequest()); + public OptionallyResolvedIndices resolveIndices(SearchTemplateRequest request) { + if (request.getRequest() != null) { + return transportSearchAction.resolveIndices(request.getRequest()); + } else { + // For the RenderSearchTemplateAction, request.getRequest() will be null. + return ResolvedIndices.unknown(); + } } } diff --git a/modules/reindex/src/test/java/org/opensearch/index/reindex/TransportReindexActionTests.java b/modules/reindex/src/test/java/org/opensearch/index/reindex/TransportReindexActionTests.java new file mode 100644 index 0000000000000..dd6a32147e7a2 --- /dev/null +++ b/modules/reindex/src/test/java/org/opensearch/index/reindex/TransportReindexActionTests.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.reindex; + +import org.opensearch.action.index.IndexAction; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.TransportSearchAction; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.script.ScriptService; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.opensearch.transport.client.Client; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportReindexActionTests extends OpenSearchTestCase { + public void testResolveIndices() { + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.getThreadContext()).thenReturn(threadContext); + TransportSearchAction transportSearchAction = mock(TransportSearchAction.class); + when(transportSearchAction.resolveIndices(any())).thenAnswer(i -> ResolvedIndices.of(((SearchRequest) i.getArgument(0)).indices())); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext); + + TransportReindexAction action = new TransportReindexAction( + Settings.EMPTY, + threadPool, + mock(ActionFilters.class), + indexNameExpressionResolver, + mock(ClusterService.class), + mock(ScriptService.class), + null, + mock(Client.class), + mock(TransportService.class), + mock(ReindexSslConfig.class), + transportSearchAction + ); + + { + ResolvedIndices resolvedIndices = action.resolveIndices( + new ReindexRequest(new SearchRequest("searched-index"), new IndexRequest("target-index")) + ); + assertEquals( + ResolvedIndices.of("searched-index").withLocalSubActions(IndexAction.INSTANCE, ResolvedIndices.Local.of("target-index")), + resolvedIndices + ); + } + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index a3807cee6efd9..da99c98a4e403 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -218,16 +218,27 @@ public ResolvedIndices resolveIndices(GetAliasesRequest request) { // containing one of the specified indices // - both aliases and indices specified: this is then the intersection // + // For both indices and aliases, patterns can be specified. The indicesOptions of the request only apply to the indices. + // // The consequence for the resolveIndices() method is that the semantics of the return value might be debatable. - // We resort to just the index names, because it is the most precise dimension. If we would include alias names, - // these could also refer to indices which were not requested. + // We resort to have just the index names on the top level, because it is the most precise dimension. + // Alias names are included as a sub action named "indices:admin/aliases/get[aliases]" String[] concreteIndices; try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().newStoredContext(false)) { - concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request); + concreteIndices = indexNameExpressionResolver.concreteResolvedIndices(state, request) + .withoutResolutionErrors() + .namesOfConcreteIndicesAsArray(); } + Map> indexToAliasesMap = state.metadata().findAliases(request, concreteIndices); - return ResolvedIndices.of(state.metadata().findAliases(request, concreteIndices).keySet()); + return ResolvedIndices.of(indexToAliasesMap.keySet()) + .withLocalSubActions( + GetAliasesAction.NAME + "[aliases]", + ResolvedIndices.Local.of( + indexToAliasesMap.values().stream().flatMap(l -> l.stream()).map(alias -> alias.alias()).collect(Collectors.toSet()) + ) + ); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java index 31d3c1373b3c4..d6bd0f5254413 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -32,6 +32,8 @@ package org.opensearch.action.admin.indices.create; +import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.action.admin.indices.alias.IndicesAliasesAction; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; @@ -50,6 +52,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.stream.Collectors; /** * Create index action. @@ -149,7 +152,16 @@ protected void clusterManagerOperation( @Override public ResolvedIndices resolveIndices(CreateIndexRequest request) { - return ResolvedIndices.of(resolveIndexName(request)); + ResolvedIndices result = ResolvedIndices.of(resolveIndexName(request)); + + if (request.aliases().isEmpty()) { + return result; + } else { + return result.withLocalSubActions( + IndicesAliasesAction.INSTANCE, + ResolvedIndices.Local.of(request.aliases().stream().map(Alias::name).collect(Collectors.toSet())) + ); + } } private String resolveIndexName(CreateIndexRequest request) { diff --git a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java index 4a934002e9f81..329c542f39216 100644 --- a/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java +++ b/server/src/main/java/org/opensearch/action/support/single/shard/TransportSingleShardAction.java @@ -160,7 +160,7 @@ protected void resolveRequest(ClusterState state, InternalRequest request) { @Override public ResolvedIndices resolveIndices(Request request) { - return ResolvedIndices.of(resolveToConcreteSingleIndex(request, clusterService.state())); + return ResolvedIndices.ofNonNull(resolveToConcreteSingleIndex(request, clusterService.state())); } private String resolveToConcreteSingleIndex(Request request, ClusterState clusterState) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java index 5492410ec7041..b55214d84602f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ResolvedIndices.java @@ -48,10 +48,22 @@ */ @ExperimentalApi public class ResolvedIndices extends OptionallyResolvedIndices { + + /** + * Creates a ResolvedIndices object with a ResolvedIndices.Local object that returns the given indicesAliasesAndDataStreams + * as names(). + * + * @param indicesAliasesAndDataStreams An array of strings. The strings must be not null. No further validation will be performed. + */ public static ResolvedIndices of(String... indicesAliasesAndDataStreams) { return new ResolvedIndices(new Local(Set.of(indicesAliasesAndDataStreams), null, Map.of()), Remote.EMPTY); } + /** + * Creates a ResolvedIndices object with a ResolvedIndices.Local.Concrete object that returns the given indices + * by its concreteIndices() and namesOfConcreteIndices() method. The names() method will return the same names as + * namesOfConcreteIndices(). + */ public static ResolvedIndices of(Index... indices) { return new ResolvedIndices( new Local.Concrete( @@ -65,18 +77,37 @@ public static ResolvedIndices of(Index... indices) { ); } + /** + * Creates a ResolvedIndices object with a ResolvedIndices.Local object that returns the given indicesAliasesAndDataStreams + * as names(). + * + * @param indicesAliasesAndDataStreams A collection of strings. The strings must be not null. No further validation will be performed. + */ public static ResolvedIndices of(Collection indicesAliasesAndDataStreams) { return new ResolvedIndices(new Local(Set.copyOf(indicesAliasesAndDataStreams), null, Map.of()), Remote.EMPTY); } + /** + * Creates a ResolvedIndices object where local() returns the given Local object. + */ public static ResolvedIndices of(Local local) { return new ResolvedIndices(local, Remote.EMPTY); } + /** + * Returns an OptionallyResolvedIndices object which is not a ResolvedIndices object. This represents the case + * that indices cannot be determined. + */ public static OptionallyResolvedIndices unknown() { return OptionallyResolvedIndices.unknown(); } + /** + * Creates a ResolvedIndices object with a ResolvedIndices.Local object that returns the given indicesAliasesAndDataStreams + * as names(). + * + * @param indicesAliasesAndDataStreams An array of strings. Any string that is null will be not included in this object. No further validation will be performed. + */ public static ResolvedIndices ofNonNull(String... indicesAliasesAndDataStreams) { Set indexSet = new HashSet<>(indicesAliasesAndDataStreams.length); @@ -106,6 +137,9 @@ public Remote remote() { return this.remote; } + /** + * Creates a new copy of this ResolvedIndices object which has the given remote indices associated. + */ public ResolvedIndices withRemoteIndices(Map remoteIndices) { if (remoteIndices.isEmpty()) { return this; @@ -125,10 +159,25 @@ public ResolvedIndices withLocalOriginalIndices(OriginalIndices originalIndices) return new ResolvedIndices(this.local.withOriginalIndices(originalIndices), this.remote); } + /** + * Creates a new copy of this ResolvedIndices object which has the given ResolvedIndices.Local instance associated + * by the given actionType. + */ public ResolvedIndices withLocalSubActions(ActionType actionType, ResolvedIndices.Local local) { return new ResolvedIndices(this.local.withSubActions(actionType, local), this.remote); } + /** + * Creates a new copy of this ResolvedIndices object which has the given ResolvedIndices.Local instance associated + * by the given actionType. + */ + public ResolvedIndices withLocalSubActions(String actionType, ResolvedIndices.Local local) { + return new ResolvedIndices(this.local.withSubActions(actionType, local), this.remote); + } + + /** + * Returns true if both the local and the remote indices do not refer to any names (existing or non-existing). + */ public boolean isEmpty() { return this.local.isEmpty() && this.remote.isEmpty(); } @@ -162,12 +211,22 @@ public static class Local extends OptionallyResolvedIndices.Local { protected final Map subActions; private Set namesOfIndices; - public static Local of(Collection names) { - return new Local(Set.copyOf(names), null, Map.of()); + /** + * Creates a ResolvedIndices.Local object that returns the given indicesAliasesAndDataStreams as names(). + * + * @param indicesAliasesAndDataStreams A collection of strings. The strings must be not null. No further validation will be performed. + */ + public static Local of(Collection indicesAliasesAndDataStreams) { + return new Local(Set.copyOf(indicesAliasesAndDataStreams), null, Map.of()); } - public static Local of(String... names) { - return of(Arrays.asList(names)); + /** + * Creates a ResolvedIndices.Local object that returns the given indicesAliasesAndDataStreams as names(). + * + * @param indicesAliasesAndDataStreams An array of strings. The strings must be not null. No further validation will be performed. + */ + public static Local of(String... indicesAliasesAndDataStreams) { + return of(Arrays.asList(indicesAliasesAndDataStreams)); } /** @@ -398,7 +457,12 @@ private Concrete( /** * Returns the concrete indices. This might throw an exception if there were issues during index resolution. - * If you need access to index information while avoiding exceptions, use the names() method instead. + *

+ * If you need access to index information while avoiding exceptions, you can use the following approaches: + *

    + *
  • Use the names() method instead. This might also contain of non-existing indices or invalid names
    • + *
  • Use withoutResolutionErrors().concreteIndices(). This will only contain existing indices.
    • + *
* * @throws org.opensearch.index.IndexNotFoundException This exception is thrown for several conditions: * 1. If one of the index expression pointed to a missing index, alias or data stream and the IndicesOptions @@ -412,14 +476,62 @@ public Set concreteIndices() { return this.concreteIndices; } + /** + * Returns the concrete indices. This might throw an exception if there were issues during index resolution. + *

+ * If you need access to index information while avoiding exceptions, you can use the following approaches: + *

    + *
  • Use the names() method instead. This might also contain of non-existing indices or invalid names
    • + *
  • Use withoutResolutionErrors().concreteIndices(). This will only contain existing indices.
    • + *
+ * + * @throws org.opensearch.index.IndexNotFoundException This exception is thrown for several conditions: + * 1. If one of the index expression pointed to a missing index, alias or data stream and the IndicesOptions + * used during resolution do not allow such a case. 2. If the set of resolved concrete indices is empty and + * the IndicesOptions used during resolution do not allow such a case. + * @throws IllegalArgumentException if one of the aliases resolved to multiple indices and + * IndicesOptions.FORBID_ALIASES_TO_MULTIPLE_INDICES was set. + */ public Index[] concreteIndicesAsArray() { return this.concreteIndices().toArray(Index.EMPTY_ARRAY); } + /** + * Returns the concrete indices. This might throw an exception if there were issues during index resolution. + *

+ * If you need access to index information while avoiding exceptions, you can use the following approaches: + *

    + *
  • Use the names() method instead. This might also contain of non-existing indices or invalid names
    • + *
  • Use withoutResolutionErrors().concreteIndices(). This will only contain existing indices.
    • + *
+ * + * @throws org.opensearch.index.IndexNotFoundException This exception is thrown for several conditions: + * 1. If one of the index expression pointed to a missing index, alias or data stream and the IndicesOptions + * used during resolution do not allow such a case. 2. If the set of resolved concrete indices is empty and + * the IndicesOptions used during resolution do not allow such a case. + * @throws IllegalArgumentException if one of the aliases resolved to multiple indices and + * IndicesOptions.FORBID_ALIASES_TO_MULTIPLE_INDICES was set. + */ public Set namesOfConcreteIndices() { return this.concreteIndices().stream().map(Index::getName).collect(Collectors.toSet()); } + /** + * Returns the concrete indices. This might throw an exception if there were issues during index resolution. + *

+ * If you need access to index information while avoiding exceptions, you can use the following approaches: + *

    + *
  • Use the names() method instead. This might also contain of non-existing indices or invalid names
    • + *
  • Use withoutResolutionErrors().concreteIndices(). This will only contain existing indices.
    • + *
+ * + * @throws org.opensearch.index.IndexNotFoundException This exception is thrown for several conditions: + * 1. If one of the index expression pointed to a missing index, alias or data stream and the IndicesOptions + * used during resolution do not allow such a case. 2. If the set of resolved concrete indices is empty and + * the IndicesOptions used during resolution do not allow such a case. + * @throws IllegalArgumentException if one of the aliases resolved to multiple indices and + * IndicesOptions.FORBID_ALIASES_TO_MULTIPLE_INDICES was set. + */ public String[] namesOfConcreteIndicesAsArray() { return this.concreteIndices().stream().map(Index::getName).toArray(String[]::new); } @@ -436,6 +548,10 @@ public ResolvedIndices.Local withSubActions(String actionType, ResolvedIndices.L return new Concrete(this.concreteIndices, this.names, this.originalIndices, subActions, resolutionErrors); } + /** + * Returns a new copy of this ResolvedIndices.Local.Concrete instances which has the given resolutionErrors + * associated. + */ public ResolvedIndices.Local.Concrete withResolutionErrors(List resolutionErrors) { if (resolutionErrors.isEmpty()) { return this; @@ -450,10 +566,18 @@ public ResolvedIndices.Local.Concrete withResolutionErrors(List clusterToOriginalIndicesMap, List asClusterToOriginalIndicesMap() { return this.clusterToOriginalIndicesMap; } + /** + * Returns the remote clusters in the format ["remote_1:index_1", "remote_1:index_2", "remote_2:index_3"] + */ public List asRawExpressions() { List result = this.rawExpressions; @@ -526,10 +656,16 @@ public List asRawExpressions() { return result; } + /** + * Returns the remote clusters in the format ["remote_1:index_1", "remote_1:index_2", "remote_2:index_3"] + */ public String[] asRawExpressionsArray() { return this.asRawExpressions().toArray(new String[0]); } + /** + * Returns true if no remote indices are present + */ public boolean isEmpty() { return this.clusterToOriginalIndicesMap.isEmpty(); } @@ -554,11 +690,7 @@ public int hashCode() { } private List buildRawExpressions() { - if (this.clusterToOriginalIndicesMap.isEmpty()) { - return Collections.emptyList(); - } - - List result = new ArrayList<>(); + List result = new ArrayList<>(this.clusterToOriginalIndicesMap.size()); for (Map.Entry entry : this.clusterToOriginalIndicesMap.entrySet()) { for (String remoteIndex : entry.getValue().indices()) { @@ -569,5 +701,4 @@ private List buildRawExpressions() { return Collections.unmodifiableList(result); } } - } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index 97fd2ff52bc13..ce02562f2df62 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -271,13 +271,19 @@ public void testResolveIndices() { { // Request an alias: Return the indices in the alias ResolvedIndices resolvedIndices = action.resolveIndices(new GetAliasesRequest("d")); - assertEquals(ResolvedIndices.of("c"), resolvedIndices); + assertEquals( + ResolvedIndices.of("c").withLocalSubActions("indices:admin/aliases/get[aliases]", ResolvedIndices.Local.of("d")), + resolvedIndices + ); } { // Request an index: Return the index itself ResolvedIndices resolvedIndices = action.resolveIndices(new GetAliasesRequest().indices("c")); - assertEquals(ResolvedIndices.of("c"), resolvedIndices); + assertEquals( + ResolvedIndices.of("c").withLocalSubActions("indices:admin/aliases/get[aliases]", ResolvedIndices.Local.of("d")), + resolvedIndices + ); } } diff --git a/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java index 5880de71f3910..68f639bb674d2 100644 --- a/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java @@ -388,4 +388,12 @@ public void testResolveIndices_concreteIndex() { ResolvedIndices resolvedIndices = action.resolveIndices(request); assertEquals(ResolvedIndices.of("concrete_value"), resolvedIndices); } + + public void testResolveIndices_nonExisting() { + Request request = new Request().index("test_non_existing"); + request.shardId = new ShardId("test_non_existing", "_na_", 0); + setState(clusterService, ClusterStateCreationUtils.state("test", true, ShardRoutingState.STARTED)); + ResolvedIndices resolvedIndices = action.resolveIndices(request); + assertEquals(ResolvedIndices.of("test_non_existing"), resolvedIndices); + } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java index e129cbf1fa02b..0a1e6d7e803c3 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java @@ -80,6 +80,7 @@ public void testUnknown() { assertTrue(resolvedIndices.local().contains("whatever")); assertTrue(resolvedIndices.local().containsAny(List.of("whatever"))); assertTrue(resolvedIndices.local().containsAny(i -> false)); + assertEquals("ResolvedIndices{unknown=true}", resolvedIndices.toString()); } public void testWithRemoteIndices() { @@ -137,6 +138,17 @@ public void testLocalOf() { assertThat(local.names(), containsInAnyOrder("o", "p", "q")); } + public void testNamesWithClusterState() { + Metadata.Builder metadata = Metadata.builder() + .put(indexBuilder("index_a1").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_a2").putAlias(AliasMetadata.builder("alias_a"))) + .put(indexBuilder("index_b1")); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); + ResolvedIndices resolvedIndices = ResolvedIndices.of("index_not_existing", "alias_a", "index_b1"); + assertThat(resolvedIndices.local().names(clusterState), containsInAnyOrder("index_not_existing", "alias_a", "index_b1")); + assertThat(ResolvedIndices.unknown().local().names(clusterState), containsInAnyOrder("index_a1", "index_a2", "index_b1")); + } + public void testNamesOfIndices() { Metadata.Builder metadata = Metadata.builder() .put(indexBuilder("index_a1").putAlias(AliasMetadata.builder("alias_a"))) @@ -176,6 +188,8 @@ public void testEqualsHashCode() { assertNotEquals(resolvedIndices1.hashCode(), resolvedIndices3.hashCode()); assertFalse(resolvedIndices1.equals(resolvedIndices3)); assertFalse(resolvedIndices3.equals(resolvedIndices1)); + assertFalse(resolvedIndices1.equals(ResolvedIndices.unknown())); + assertNotEquals(resolvedIndices1.hashCode(), ResolvedIndices.unknown().hashCode()); } public void testHashCodeForConcreteIndices() { From b0719f3dc74040d834ebc0b6db70fea26b704a2f Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 9 Sep 2025 11:55:40 +0200 Subject: [PATCH 16/24] Fix Signed-off-by: Nils Bandener --- .../opensearch/cluster/metadata/ResolvedIndicesTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java index 0a1e6d7e803c3..32399e89578f0 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ResolvedIndicesTests.java @@ -146,7 +146,10 @@ public void testNamesWithClusterState() { ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build(); ResolvedIndices resolvedIndices = ResolvedIndices.of("index_not_existing", "alias_a", "index_b1"); assertThat(resolvedIndices.local().names(clusterState), containsInAnyOrder("index_not_existing", "alias_a", "index_b1")); - assertThat(ResolvedIndices.unknown().local().names(clusterState), containsInAnyOrder("index_a1", "index_a2", "index_b1")); + assertThat( + ResolvedIndices.unknown().local().names(clusterState), + containsInAnyOrder("index_a1", "index_a2", "index_b1", "alias_a") + ); } public void testNamesOfIndices() { From ec8bab69247fc8cfebe04e8faed5e706555f1642 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Thu, 11 Sep 2025 23:38:28 +0200 Subject: [PATCH 17/24] Additional actions Signed-off-by: Nils Bandener --- .../alias/get/TransportGetAliasesAction.java | 2 +- .../datastream/DataStreamsStatsAction.java | 33 +++-- .../indices/resolve/ResolveIndexAction.java | 34 ++++- .../metadata/IndexAbstractionResolver.java | 40 ++++-- .../TransportCreateIndexActionTests.java | 31 ++-- .../DataStreamStatsActionTests.java | 68 +++++++++ .../resolve/ResolveIndexActionTests.java | 133 ++++++++++++++++++ .../indices/resolve/ResolveIndexTests.java | 3 +- ...ransportIndicesShardStoresActionTests.java | 63 +++++++++ .../get/TransportMultiGetActionTests.java | 16 +++ .../search/TransportDeletePitActionTests.java | 120 +++++++--------- 11 files changed, 442 insertions(+), 101 deletions(-) create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamStatsActionTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexActionTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresActionTests.java diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index da99c98a4e403..8f701ed7a7412 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -233,7 +233,7 @@ public ResolvedIndices resolveIndices(GetAliasesRequest request) { } Map> indexToAliasesMap = state.metadata().findAliases(request, concreteIndices); - return ResolvedIndices.of(indexToAliasesMap.keySet()) + return ResolvedIndices.of(concreteIndices) .withLocalSubActions( GetAliasesAction.NAME + "[aliases]", ResolvedIndices.Local.of( diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java index fcb13f4091638..e7194583912ee 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DataStreamsStatsAction.java @@ -47,6 +47,7 @@ import org.opensearch.cluster.metadata.IndexAbstractionResolver; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; @@ -412,16 +413,7 @@ protected ClusterBlockException checkRequestBlock(ClusterState state, Request re @Override protected ShardsIterator shards(ClusterState clusterState, Request request, String[] concreteIndices) { - String[] requestIndices = request.indices(); - if (requestIndices == null || requestIndices.length == 0) { - requestIndices = new String[] { "*" }; - } - List abstractionNames = indexAbstractionResolver.resolveIndexAbstractions( - requestIndices, - request.indicesOptions(), - clusterState.getMetadata(), - true - ); // Always include data streams for data streams stats api + Set abstractionNames = resolvedIndices(request, clusterState, true).local().names(); SortedMap indicesLookup = clusterState.getMetadata().getIndicesLookup(); String[] concreteDatastreamIndices = abstractionNames.stream().flatMap(abstractionName -> { @@ -523,5 +515,26 @@ protected Response newResponse( dataStreamStats ); } + + @Override + public ResolvedIndices resolveIndices(Request request) { + return resolvedIndices(request, clusterService.state(), false); + } + + private ResolvedIndices resolvedIndices(Request request, ClusterState clusterState, boolean throwExceptions) { + String[] requestIndices = request.indices(); + if (requestIndices == null || requestIndices.length == 0) { + requestIndices = new String[] { "*" }; + } + return ResolvedIndices.of( + indexAbstractionResolver.resolveIndexAbstractions( + requestIndices, + request.indicesOptions(), + clusterState.getMetadata(), + true, + throwExceptions + ) + ); + } } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java index 1c21b1b9cc52c..11e01f45ff261 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -40,12 +40,14 @@ import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.TransportIndicesResolvingAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexAbstraction; import org.opensearch.cluster.metadata.IndexAbstractionResolver; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; @@ -497,7 +499,9 @@ public int hashCode() { * * @opensearch.internal */ - public static class TransportAction extends HandledTransportAction { + public static class TransportAction extends HandledTransportAction + implements + TransportIndicesResolvingAction { private final ThreadPool threadPool; private final ClusterService clusterService; @@ -574,6 +578,34 @@ protected void doExecute(Task task, Request request, final ActionListener remoteClusterIndices = remoteClusterService.groupIndices( + request.indicesOptions(), + request.indices(), + idx -> indexNameExpressionResolver.hasIndexAbstraction(idx, clusterState) + ); + OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); + + if (localIndices != null) { + return ResolvedIndices.of( + indexAbstractionResolver.resolveIndexAbstractions( + localIndices.indices(), + localIndices.indicesOptions(), + clusterState.metadata(), + request.includeDataStreams(), + false + ) + ).withRemoteIndices(remoteClusterIndices); + } else { + return ResolvedIndices.of(Collections.emptySet()).withRemoteIndices(remoteClusterIndices); + } + } + /** * Resolves the specified names and/or wildcard expressions to index abstractions. Returns results in the supplied lists. * diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexAbstractionResolver.java index a83c778a4b83a..889c70b0a7972 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexAbstractionResolver.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexAbstractionResolver.java @@ -62,14 +62,25 @@ public List resolveIndexAbstractions( Metadata metadata, boolean includeDataStreams ) { - return resolveIndexAbstractions(Arrays.asList(indices), indicesOptions, metadata, includeDataStreams); + return resolveIndexAbstractions(indices, indicesOptions, metadata, includeDataStreams, true); + } + + public List resolveIndexAbstractions( + String[] indices, + IndicesOptions indicesOptions, + Metadata metadata, + boolean includeDataStreams, + boolean throwExceptions + ) { + return resolveIndexAbstractions(Arrays.asList(indices), indicesOptions, metadata, includeDataStreams, throwExceptions); } public List resolveIndexAbstractions( Iterable indices, IndicesOptions indicesOptions, Metadata metadata, - boolean includeDataStreams + boolean includeDataStreams, + boolean throwExceptions ) { final boolean replaceWildcards = indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed(); Set availableIndexAbstractions = metadata.getIndicesLookup().keySet(); @@ -79,7 +90,8 @@ public List resolveIndexAbstractions( metadata, availableIndexAbstractions, replaceWildcards, - includeDataStreams + includeDataStreams, + throwExceptions ); } @@ -89,7 +101,8 @@ public List resolveIndexAbstractions( Metadata metadata, Collection availableIndexAbstractions, boolean replaceWildcards, - boolean includeDataStreams + boolean includeDataStreams, + boolean throwExceptions ) { List finalIndices = new ArrayList<>(); boolean wildcardSeen = false; @@ -110,18 +123,29 @@ public List resolveIndexAbstractions( if (replaceWildcards && Regex.isSimpleMatchPattern(dateMathName)) { // continue indexAbstraction = dateMathName; - } else if (availableIndexAbstractions.contains(dateMathName) - && isIndexVisible(indexAbstraction, dateMathName, indicesOptions, metadata, includeDataStreams, true)) { + } else if (availableIndexAbstractions.contains(dateMathName)) { + if (isIndexVisible(indexAbstraction, dateMathName, indicesOptions, metadata, includeDataStreams, true)) { if (minus) { finalIndices.remove(dateMathName); } else { finalIndices.add(dateMathName); } - } else { + } else if (throwExceptions) { if (indicesOptions.ignoreUnavailable() == false) { throw new IndexNotFoundException(dateMathName); } } + } else { + if (!throwExceptions) { + if (minus) { + finalIndices.remove(dateMathName); + } else { + finalIndices.add(dateMathName); + } + } else if (indicesOptions.ignoreUnavailable() == false) { + throw new IndexNotFoundException(dateMathName); + } + } } if (replaceWildcards && Regex.isSimpleMatchPattern(indexAbstraction)) { @@ -135,7 +159,7 @@ && isIndexVisible(indexAbstraction, authorizedIndex, indicesOptions, metadata, i } if (resolvedIndices.isEmpty()) { // es core honours allow_no_indices for each wildcard expression, we do the same here by throwing index not found. - if (indicesOptions.allowNoIndices() == false) { + if (indicesOptions.allowNoIndices() == false && throwExceptions) { throw new IndexNotFoundException(indexAbstraction); } } else { diff --git a/server/src/test/java/org/opensearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/create/TransportCreateIndexActionTests.java index 5b5c8e5954157..6e09c452ff53e 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -8,11 +8,16 @@ package org.opensearch.action.admin.indices.create; +import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.action.admin.indices.alias.IndicesAliasesAction; import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilters; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MetadataCreateIndexService; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; import org.opensearch.index.mapper.MappingTransformerRegistry; import org.opensearch.test.OpenSearchTestCase; @@ -40,8 +45,6 @@ public class TransportCreateIndexActionTests extends OpenSearchTestCase { private ThreadPool threadPool; @Mock private MetadataCreateIndexService createIndexService; - @Mock - private IndexNameExpressionResolver indexNameExpressionResolver; @Mock private MappingTransformerRegistry mappingTransformerRegistry; @@ -66,14 +69,12 @@ public void setup() { threadPool, createIndexService, actionFilters, - indexNameExpressionResolver, + new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), mappingTransformerRegistry ); } public void testClusterManagerOperation_usesTransformedMapping() { - when(indexNameExpressionResolver.resolveDateMathExpression(any())).thenReturn("test-index"); - // Arrange: Create a test request final CreateIndexRequest request = new CreateIndexRequest("test-index"); request.mapping("{\"properties\": {\"field\": {\"type\": \"text\"}}}"); @@ -83,11 +84,7 @@ public void testClusterManagerOperation_usesTransformedMapping() { // Capture ActionListener passed to applyTransformers final ArgumentCaptor> listenerCaptor = ArgumentCaptor.forClass(ActionListener.class); - doNothing().when(mappingTransformerRegistry).applyTransformers( - anyString(), - any(), - listenerCaptor.capture() - ); + doNothing().when(mappingTransformerRegistry).applyTransformers(anyString(), any(), listenerCaptor.capture()); // Act: Call the method action.clusterManagerOperation(request, clusterState, responseListener); @@ -96,12 +93,22 @@ public void testClusterManagerOperation_usesTransformedMapping() { listenerCaptor.getValue().onResponse(transformedMapping); // Assert: Capture request sent to createIndexService - ArgumentCaptor updateRequestCaptor = - ArgumentCaptor.forClass(CreateIndexClusterStateUpdateRequest.class); + ArgumentCaptor updateRequestCaptor = ArgumentCaptor.forClass( + CreateIndexClusterStateUpdateRequest.class + ); verify(createIndexService, times(1)).createIndex(updateRequestCaptor.capture(), any()); // Ensure transformed mapping is passed correctly CreateIndexClusterStateUpdateRequest capturedRequest = updateRequestCaptor.getValue(); assertEquals(transformedMapping, capturedRequest.mappings()); } + + public void testResolveIndices() { + assertEquals(ResolvedIndices.of("test_index"), action.resolveIndices(new CreateIndexRequest("test_index"))); + assertEquals( + ResolvedIndices.of("test_index").withLocalSubActions(IndicesAliasesAction.INSTANCE, ResolvedIndices.Local.of("test_alias")), + action.resolveIndices(new CreateIndexRequest("test_index").alias(new Alias("test_alias"))) + ); + } + } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamStatsActionTests.java new file mode 100644 index 0000000000000..4874ada2aa2b2 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/datastream/DataStreamStatsActionTests.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.datastream; + +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.DataStream; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.indices.IndicesService; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.transport.TransportService; + +import java.util.List; + +import static org.opensearch.cluster.DataStreamTestHelper.createBackingIndex; +import static org.opensearch.cluster.DataStreamTestHelper.createTimestampField; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DataStreamStatsActionTests extends OpenSearchTestCase { + public void testResolveIndices() throws Exception { + String dataStreamName = "datastream-test"; + + IndexMetadata index1 = createBackingIndex(dataStreamName, 1).build(); + IndexMetadata index2 = createBackingIndex(dataStreamName, 2).build(); + + Metadata.Builder mdBuilder = Metadata.builder() + .put(index1, false) + .put(index2, false) + .put(new DataStream(dataStreamName, createTimestampField("@timestamp"), List.of(index1.getIndex(), index2.getIndex()), 2)); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build(); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + + DataStreamsStatsAction.TransportAction action = new DataStreamsStatsAction.TransportAction( + clusterService, + mock(TransportService.class), + mock(IndicesService.class), + mock(ActionFilters.class), + new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) + ); + + { + ResolvedIndices resolvedIndices = action.resolveIndices(new DataStreamsStatsAction.Request().indices("datastream-*")); + assertEquals(ResolvedIndices.of("datastream-test"), resolvedIndices); + } + + { + ResolvedIndices resolvedIndices = action.resolveIndices(new DataStreamsStatsAction.Request()); + assertEquals(ResolvedIndices.of("datastream-test"), resolvedIndices); + } + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexActionTests.java new file mode 100644 index 0000000000000..9a4f333fe23cc --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexActionTests.java @@ -0,0 +1,133 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.resolve; + +import org.opensearch.Version; +import org.opensearch.action.OriginalIndices; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.telemetry.tracing.noop.NoopTracer; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.transport.MockTransportService; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.RemoteClusterConnectionTests; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ResolveIndexActionTests extends OpenSearchTestCase { + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Override + public void tearDown() throws Exception { + super.tearDown(); + ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); + } + + public void testResolveIndices() { + IndexMetadata.Builder indexA1 = IndexMetadata.builder("index_a1") + .putAlias(AliasMetadata.builder("alias_a").build()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + IndexMetadata.Builder indexA2 = IndexMetadata.builder("index_a2") + .putAlias(AliasMetadata.builder("alias_a").build()) + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().put(indexA1).put(indexA2)) + .build(); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)); + + Settings.Builder settingsBuilder = Settings.builder(); + try ( + MockTransportService remoteSeedTransport = RemoteClusterConnectionTests.startTransport( + "node_remote", + Collections.emptyList(), + Version.CURRENT, + threadPool + ) + ) { + DiscoveryNode remoteSeedNode = remoteSeedTransport.getLocalDiscoNode(); + settingsBuilder.put("cluster.remote.remote1.seeds", remoteSeedNode.getAddress().toString()); + + try ( + MockTransportService transportService = MockTransportService.createNewService( + settingsBuilder.build(), + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) + ) { + transportService.start(); + transportService.acceptIncomingRequests(); + + ResolveIndexAction.TransportAction action = new ResolveIndexAction.TransportAction( + transportService, + clusterService, + threadPool, + mock(ActionFilters.class), + indexNameExpressionResolver + ); + + // Actual test cases start here: + { + ResolvedIndices resolvedIndices = action.resolveIndices(new ResolveIndexAction.Request(new String[] { "index_*" })); + assertEquals(ResolvedIndices.of("index_a1", "index_a2"), resolvedIndices); + } + + { + ResolvedIndices resolvedIndices = action.resolveIndices(new ResolveIndexAction.Request(new String[] { "alias_a" })); + assertEquals(ResolvedIndices.of("alias_a"), resolvedIndices); + } + + { + ResolvedIndices resolvedIndices = action.resolveIndices( + new ResolveIndexAction.Request(new String[] { "index_non_existing" }) + ); + assertEquals(ResolvedIndices.of("index_non_existing"), resolvedIndices); + } + + { + ResolvedIndices resolvedIndices = action.resolveIndices( + new ResolveIndexAction.Request(new String[] { "index_*", "remote1:remote_index" }) + ); + assertEquals( + ResolvedIndices.of("index_a1", "index_a2") + .withRemoteIndices( + Map.of("remote1", new OriginalIndices(new String[] { "remote_index" }, IndicesOptions.LENIENT_EXPAND_OPEN)) + ), + resolvedIndices + ); + } + + } + } + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexTests.java index 8fbe46be856ba..6f7d82333f974 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -199,7 +199,8 @@ public void testResolveHiddenProperlyWithDateMath() { metadata, asList("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix), randomBoolean(), - randomBoolean() + randomBoolean(), + true ); assertThat(resolvedIndices.size(), is(1)); assertThat(resolvedIndices.get(0), oneOf("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix)); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresActionTests.java new file mode 100644 index 0000000000000..9426bf4cf87aa --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresActionTests.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.shards; + +import org.opensearch.Version; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.gateway.TransportNodesListGatewayStartedShards; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportIndicesShardStoresActionTests extends OpenSearchTestCase { + + public void testResolveIndices() { + IndexMetadata.Builder indexA1 = IndexMetadata.builder("index_a1") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + IndexMetadata.Builder indexA2 = IndexMetadata.builder("index_b1") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .metadata(Metadata.builder().put(indexA1).put(indexA2)) + .build(); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.getThreadContext()).thenReturn(threadContext); + + TransportIndicesShardStoresAction action = new TransportIndicesShardStoresAction( + mock(TransportService.class), + clusterService, + threadPool, + mock(ActionFilters.class), + new IndexNameExpressionResolver(threadContext), + mock(TransportNodesListGatewayStartedShards.class), + null + ); + + assertEquals(ResolvedIndices.of("index_a1"), action.resolveIndices(new IndicesShardStoresRequest("index_a*"))); + } +} diff --git a/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java b/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java index a410eaf8937cb..8ca51ead9127b 100644 --- a/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java +++ b/server/src/test/java/org/opensearch/action/get/TransportMultiGetActionTests.java @@ -41,6 +41,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.ResolvedIndices; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.OperationRouting; import org.opensearch.cluster.routing.Preference; @@ -317,7 +318,22 @@ public void testShouldForcePrimaryRouting() throws IOException { // should fail since document replication enabled assertFalse(TransportGetAction.shouldForcePrimaryRouting(metadata, true, null, "index1")); + } + + public void testResolveIndices() { + MultiGetRequest multiGetRequest = new MultiGetRequest(); + multiGetRequest.add("index1", "1"); + multiGetRequest.add("index2", "2"); + + transportAction = new TransportMultiGetAction( + transportService, + clusterService, + shardAction, + new ActionFilters(emptySet()), + new Resolver() + ); + assertEquals(ResolvedIndices.of("index1", "index2"), transportAction.resolveIndices(multiGetRequest)); } private static Task createTask() { diff --git a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java index 0a31d2ad0b352..601ecb37a3ad7 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportDeletePitActionTests.java @@ -723,43 +723,35 @@ public void testResolveIndices() throws InterruptedException, ExecutionException client.initialize(null, null, null, namedWriteableRegistry); ActionFilters actionFilters = mock(ActionFilters.class); when(actionFilters.filters()).thenReturn(new ActionFilter[0]); - List knownNodes = new CopyOnWriteArrayList<>(); + try ( - MockTransportService cluster1Transport = startTransport("cluster_1_node", knownNodes, Version.CURRENT); - MockTransportService cluster2Transport = startTransport("cluster_2_node", knownNodes, Version.CURRENT) + MockTransportService transportService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) ) { - knownNodes.add(cluster1Transport.getLocalDiscoNode()); - knownNodes.add(cluster2Transport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try ( - MockTransportService transportService = MockTransportService.createNewService( - Settings.EMPTY, - Version.CURRENT, - threadPool, - NoopTracer.INSTANCE - ) - ) { - transportService.start(); - transportService.acceptIncomingRequests(); - SearchTransportService searchTransportService = new SearchTransportService(transportService, null) { - @Override - public Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) { - return new SearchAsyncActionTests.MockConnection(node); - } - }; - PitService pitService = new PitService(clusterServiceMock, searchTransportService, transportService, client); - TransportDeletePitAction action = new TransportDeletePitAction( - transportService, - actionFilters, - namedWriteableRegistry, - pitService - ); - DeletePitRequest deletePITRequest = new DeletePitRequest(pitId); - OptionallyResolvedIndices resolvedIndices = action.resolveIndices(deletePITRequest); - assertEquals(ResolvedIndices.of("idx", "idy"), resolvedIndices); - } + transportService.start(); + transportService.acceptIncomingRequests(); + SearchTransportService searchTransportService = new SearchTransportService(transportService, null) { + @Override + public Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) { + return new SearchAsyncActionTests.MockConnection(node); + } + }; + PitService pitService = new PitService(clusterServiceMock, searchTransportService, transportService, client); + TransportDeletePitAction action = new TransportDeletePitAction( + transportService, + actionFilters, + namedWriteableRegistry, + pitService + ); + DeletePitRequest deletePITRequest = new DeletePitRequest(pitId); + OptionallyResolvedIndices resolvedIndices = action.resolveIndices(deletePITRequest); + assertEquals(ResolvedIndices.of("idx", "idy"), resolvedIndices); } + } public void testResolveIndices_allPits() throws InterruptedException, ExecutionException { @@ -767,42 +759,34 @@ public void testResolveIndices_allPits() throws InterruptedException, ExecutionE client.initialize(null, null, null, namedWriteableRegistry); ActionFilters actionFilters = mock(ActionFilters.class); when(actionFilters.filters()).thenReturn(new ActionFilter[0]); - List knownNodes = new CopyOnWriteArrayList<>(); + try ( - MockTransportService cluster1Transport = startTransport("cluster_1_node", knownNodes, Version.CURRENT); - MockTransportService cluster2Transport = startTransport("cluster_2_node", knownNodes, Version.CURRENT) + MockTransportService transportService = MockTransportService.createNewService( + Settings.EMPTY, + Version.CURRENT, + threadPool, + NoopTracer.INSTANCE + ) ) { - knownNodes.add(cluster1Transport.getLocalDiscoNode()); - knownNodes.add(cluster2Transport.getLocalDiscoNode()); - Collections.shuffle(knownNodes, random()); - - try ( - MockTransportService transportService = MockTransportService.createNewService( - Settings.EMPTY, - Version.CURRENT, - threadPool, - NoopTracer.INSTANCE - ) - ) { - transportService.start(); - transportService.acceptIncomingRequests(); - SearchTransportService searchTransportService = new SearchTransportService(transportService, null) { - @Override - public Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) { - return new SearchAsyncActionTests.MockConnection(node); - } - }; - PitService pitService = new PitService(clusterServiceMock, searchTransportService, transportService, client); - TransportDeletePitAction action = new TransportDeletePitAction( - transportService, - actionFilters, - namedWriteableRegistry, - pitService - ); - DeletePitRequest deletePITRequest = new DeletePitRequest("_all"); - OptionallyResolvedIndices resolvedIndices = action.resolveIndices(deletePITRequest); - assertEquals(ResolvedIndices.unknown(), resolvedIndices); - } + transportService.start(); + transportService.acceptIncomingRequests(); + SearchTransportService searchTransportService = new SearchTransportService(transportService, null) { + @Override + public Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) { + return new SearchAsyncActionTests.MockConnection(node); + } + }; + PitService pitService = new PitService(clusterServiceMock, searchTransportService, transportService, client); + TransportDeletePitAction action = new TransportDeletePitAction( + transportService, + actionFilters, + namedWriteableRegistry, + pitService + ); + DeletePitRequest deletePITRequest = new DeletePitRequest("_all"); + OptionallyResolvedIndices resolvedIndices = action.resolveIndices(deletePITRequest); + assertEquals(ResolvedIndices.unknown(), resolvedIndices); } + } } From 589289094d922633d8e02cbdf6e074edde5d360a Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 12 Sep 2025 08:42:52 +0200 Subject: [PATCH 18/24] Fix Signed-off-by: Nils Bandener --- .../indices/alias/get/TransportGetAliasesAction.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index 8f701ed7a7412..d18041b2208dc 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -224,6 +225,7 @@ public ResolvedIndices resolveIndices(GetAliasesRequest request) { // We resort to have just the index names on the top level, because it is the most precise dimension. // Alias names are included as a sub action named "indices:admin/aliases/get[aliases]" + boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0; String[] concreteIndices; try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().newStoredContext(false)) { @@ -233,11 +235,16 @@ public ResolvedIndices resolveIndices(GetAliasesRequest request) { } Map> indexToAliasesMap = state.metadata().findAliases(request, concreteIndices); - return ResolvedIndices.of(concreteIndices) + // If no aliases were specified in the request, we will report all indices matching the indices expression + // If there were aliases specified, we will report just the indices that are members of the aliases + // That follows the logic in the postProcess() method above. + Collection indices = noAliasesSpecified ? Arrays.asList(concreteIndices) : indexToAliasesMap.keySet(); + + return ResolvedIndices.of(indices) .withLocalSubActions( GetAliasesAction.NAME + "[aliases]", ResolvedIndices.Local.of( - indexToAliasesMap.values().stream().flatMap(l -> l.stream()).map(alias -> alias.alias()).collect(Collectors.toSet()) + indexToAliasesMap.values().stream().flatMap(Collection::stream).map(AliasMetadata::alias).collect(Collectors.toSet()) ) ); } From 0d4f2f6f6fd7587ecbcc38a6fc4382b2ac537f58 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 12 Sep 2025 10:18:42 +0200 Subject: [PATCH 19/24] Do not cache the ResolvedIndices object Signed-off-by: Nils Bandener --- .../action/support/ActionRequestMetadata.java | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java b/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java index f999d34b16124..2f9be658a5aea 100644 --- a/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java +++ b/server/src/main/java/org/opensearch/action/support/ActionRequestMetadata.java @@ -34,8 +34,6 @@ public static A private final TransportAction transportAction; private final Request request; - private OptionallyResolvedIndices resolvedIndices; - ActionRequestMetadata(TransportAction transportAction, Request request) { this.transportAction = transportAction; this.request = request; @@ -54,23 +52,10 @@ public OptionallyResolvedIndices resolvedIndices() { return OptionallyResolvedIndices.unknown(); } - if (this.resolvedIndices != null) { - return this.resolvedIndices; - } else { - return resolveIndices(); - } - } - - /** - * Performs the actual index resolution. Index resolution can be relatively costly on big clusters, so we - * perform it lazily only when requested. - */ - private OptionallyResolvedIndices resolveIndices() { + // We should not cache and re-use results in this object, as each ActionFilter might modify the request + // and thus change the result @SuppressWarnings("unchecked") TransportIndicesResolvingAction indicesResolvingAction = (TransportIndicesResolvingAction) this.transportAction; - OptionallyResolvedIndices result = indicesResolvingAction.resolveIndices(request); - - this.resolvedIndices = result; - return result; + return indicesResolvingAction.resolveIndices(request); } } From 27486935d23e5d8d149561eaeaaf7e401b5fb120 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 12 Sep 2025 11:10:09 +0200 Subject: [PATCH 20/24] Do not cache the ResolvedIndices object Signed-off-by: Nils Bandener --- .../opensearch/action/support/ActionRequestMetadataTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/action/support/ActionRequestMetadataTests.java b/server/src/test/java/org/opensearch/action/support/ActionRequestMetadataTests.java index b16a2b7a2801d..f19c339e68159 100644 --- a/server/src/test/java/org/opensearch/action/support/ActionRequestMetadataTests.java +++ b/server/src/test/java/org/opensearch/action/support/ActionRequestMetadataTests.java @@ -27,7 +27,6 @@ public void testResolvedIndices() { ); OptionallyResolvedIndices resolvedIndices = subject.resolvedIndices(); assertEquals(ResolvedIndices.of("my_resolution_result"), resolvedIndices); - assertSame("ResolvedIndices object is reused on subsequent calls", resolvedIndices, subject.resolvedIndices()); } public void testResolvedIndices_unknown() { From 72fe3f327521ff1c44102089ee94e52529b03911 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 10 Oct 2025 15:24:48 +0200 Subject: [PATCH 21/24] Added changelog Signed-off-by: Nils Bandener --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 821adfa63433e..215bb426926a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add pluggable gRPC interceptors with explicit ordering([#19005](https://github.com/opensearch-project/OpenSearch/pull/19005)) - Add metrics for the merged segment warmer feature ([#18929](https://github.com/opensearch-project/OpenSearch/pull/18929)) - Add pointer based lag metric in pull-based ingestion ([#19635](https://github.com/opensearch-project/OpenSearch/pull/19635)) +- Introduced internal API for retrieving metadata about requested indices from transport actions ([#18523](https://github.com/opensearch-project/OpenSearch/pull/18523)) ### Changed - Faster `terms` query creation for `keyword` field with index and docValues enabled ([#19350](https://github.com/opensearch-project/OpenSearch/pull/19350)) From aeef9c741287561dcf08c600c3ba554652eea281 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 22 Oct 2025 16:33:33 +0200 Subject: [PATCH 22/24] Moved to non-breaking transition of ActionFilter interface Signed-off-by: Nils Bandener --- .../action/support/ActionFilter.java | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/support/ActionFilter.java b/server/src/main/java/org/opensearch/action/support/ActionFilter.java index 0c7f5a6e220a0..8df6cc7a884c6 100644 --- a/server/src/main/java/org/opensearch/action/support/ActionFilter.java +++ b/server/src/main/java/org/opensearch/action/support/ActionFilter.java @@ -52,15 +52,38 @@ public interface ActionFilter { /** * Enables filtering the execution of an action on the request side, either by sending a response through the * {@link ActionListener} or by continuing the execution through the given {@link ActionFilterChain chain} + * + * @apiNote This has only for a transitional period a default implementation. After the deprecated apply() + * method has been removed, the default implementation will be removed. */ - void apply( + default void apply( Task task, String action, Request request, ActionRequestMetadata actionRequestMetadata, ActionListener listener, ActionFilterChain chain - ); + ) { + this.apply(task, action, request, listener, chain); + } + + /** + * Enables filtering the execution of an action on the request side, either by sending a response through the + * {@link ActionListener} or by continuing the execution through the given {@link ActionFilterChain chain} + * + * @deprecated please do not override this method any more. Instead, override the method with the additional + * ActionRequestMetadata parameter. This method will be removed soon. + */ + @Deprecated + default void apply( + Task task, + String action, + Request request, + ActionListener listener, + ActionFilterChain chain + ) { + assert false : "Incomplete implementation of ActionFilter interface in " + this; + } /** * A simple base class for injectable action filters that spares the implementation from handling the From 9f3791e135f50276a1fd9323606fa4917e29072c Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 22 Oct 2025 16:42:45 +0200 Subject: [PATCH 23/24] Fixed JavaDoc Signed-off-by: Nils Bandener --- .../main/java/org/opensearch/action/support/ActionFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/action/support/ActionFilter.java b/server/src/main/java/org/opensearch/action/support/ActionFilter.java index 8df6cc7a884c6..bf110e46dee5f 100644 --- a/server/src/main/java/org/opensearch/action/support/ActionFilter.java +++ b/server/src/main/java/org/opensearch/action/support/ActionFilter.java @@ -53,7 +53,7 @@ public interface ActionFilter { * Enables filtering the execution of an action on the request side, either by sending a response through the * {@link ActionListener} or by continuing the execution through the given {@link ActionFilterChain chain} * - * @apiNote This has only for a transitional period a default implementation. After the deprecated apply() + * Note: This has only for a transitional period a default implementation. After the deprecated apply() * method has been removed, the default implementation will be removed. */ default void apply( From 1f681db31766ef0ea7647b87e229c97c55a6e03f Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 22 Oct 2025 18:17:12 +0200 Subject: [PATCH 24/24] Fixes acc to PR comments Signed-off-by: Nils Bandener --- .../main/java/org/opensearch/action/support/ActionFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/support/ActionFilter.java b/server/src/main/java/org/opensearch/action/support/ActionFilter.java index bf110e46dee5f..1d4be8c17efc1 100644 --- a/server/src/main/java/org/opensearch/action/support/ActionFilter.java +++ b/server/src/main/java/org/opensearch/action/support/ActionFilter.java @@ -74,7 +74,7 @@ default void ap * @deprecated please do not override this method any more. Instead, override the method with the additional * ActionRequestMetadata parameter. This method will be removed soon. */ - @Deprecated + @Deprecated(forRemoval = true) default void apply( Task task, String action, @@ -82,7 +82,7 @@ default void ap ActionListener listener, ActionFilterChain chain ) { - assert false : "Incomplete implementation of ActionFilter interface in " + this; + throw new UnsupportedOperationException("Incomplete implementation of ActionFilter interface in " + this); } /**