Skip to content

Implement formatted doc values for exponential histograms #138095

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 17, 2025
Merged

Implement formatted doc values for exponential histograms #138095

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b48850e
Implement formatted doc values for exponential histograms
JonasKunz Nov 14, 2025
3fee7dc
[CI] Auto commit changes from spotless
Nov 14, 2025
5975afc
Merge remote-tracking branch 'elastic/main' into exp-histo-formattedd…
JonasKunz Nov 14, 2025
ae23c4e
Merge branch 'main' into exp-histo-formatteddocvals
JonasKunz Nov 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions ...in/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent;
import org.elasticsearch.exponentialhistogram.ZeroBucket;
import org.elasticsearch.index.fielddata.FieldDataContext;
import org.elasticsearch.index.fielddata.FormattedDocValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
import org.elasticsearch.index.mapper.BlockLoader;
Expand Down Expand Up @@ -276,6 +277,11 @@ public SortedBinaryDocValues getBytesValues() {
);
}

@Override
public FormattedDocValues getFormattedValues(DocValueFormat format) {
return createFormattedDocValues(context.reader(), fieldName);
}

@Override
public long ramBytesUsed() {
return 0; // No dynamic allocations
Expand Down Expand Up @@ -415,6 +421,43 @@ public void read(int docId, StoredFields storedFields, Builder builder) throws I
}
}

// Visible for testing
static FormattedDocValues createFormattedDocValues(LeafReader reader, String fieldName) {
return new FormattedDocValues() {

boolean hasNext = false;
ExponentialHistogramValuesReader delegate;

private ExponentialHistogramValuesReader lazyDelegate() throws IOException {
if (delegate == null) {
delegate = new DocValuesReader(reader, fieldName);
}
return delegate;
}

@Override
public boolean advanceExact(int docId) throws IOException {
hasNext = lazyDelegate().advanceExact(docId);
return hasNext;
}

@Override
public int docValueCount() throws IOException {
return 1; // no multivalue support, so always 1
}

@Override
public Object nextValue() throws IOException {
if (hasNext == false) {
throw new IllegalStateException("No value available, make sure to call advanceExact() first");
}
hasNext = false;
return lazyDelegate().histogramValue();
}

};
}

@Override
protected boolean supportsParsingObject() {
return true;
Expand Down
58 changes: 58 additions & 0 deletions ...va/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,26 @@

package org.elasticsearch.xpack.exponentialhistogram;

import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.core.Types;
import org.elasticsearch.exponentialhistogram.CompressedExponentialHistogram;
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramTestUtils;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramUtils;
import org.elasticsearch.exponentialhistogram.ZeroBucket;
import org.elasticsearch.index.fielddata.FormattedDocValues;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentParsingException;
import org.elasticsearch.index.mapper.MappedFieldType;
Expand All @@ -30,6 +42,7 @@
import org.elasticsearch.xpack.analytics.mapper.HistogramParser;
import org.elasticsearch.xpack.analytics.mapper.IndexWithCount;
import org.elasticsearch.xpack.analytics.mapper.ParsedHistogramConverter;
import org.elasticsearch.xpack.exponentialhistogram.aggregations.ExponentialHistogramAggregatorTestCase;
import org.junit.AssumptionViolatedException;
import org.junit.Before;

Expand Down Expand Up @@ -685,6 +698,51 @@ public List<SyntheticSourceInvalidExample> invalidExample() {
};
}

public void testFormattedDocValues() throws IOException {
try (Directory directory = newDirectory()) {
ExponentialHistogramCircuitBreaker noopBreaker = ExponentialHistogramCircuitBreaker.noop();

List<? extends ExponentialHistogram> inputHistograms = IntStream.range(0, randomIntBetween(1, 100))
.mapToObj(i -> ExponentialHistogramTestUtils.randomHistogram(noopBreaker))
.map(
histo -> ExponentialHistogram.builder(histo, noopBreaker)
// make sure we have a double-based zero bucket, as we can only serialize those exactly
.zeroBucket(ZeroBucket.create(histo.zeroBucket().zeroThreshold(), histo.zeroBucket().count()))
.build()
)
.map(histogram -> randomBoolean() ? null : histogram)
.toList();

IndexWriterConfig config = LuceneTestCase.newIndexWriterConfig(random(), new MockAnalyzer(random()));
RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, config);
inputHistograms.forEach(histo -> ExponentialHistogramAggregatorTestCase.addHistogramDoc(indexWriter, "field", histo));
indexWriter.close();

try (DirectoryReader reader = DirectoryReader.open(directory)) {
for (int i = 0; i < reader.leaves().size(); i++) {
LeafReaderContext leaf = reader.leaves().get(i);
int docBase = leaf.docBase;
LeafReader leafReader = leaf.reader();
int maxDoc = leafReader.maxDoc();
FormattedDocValues docValues = ExponentialHistogramFieldMapper.createFormattedDocValues(leafReader, "field");
for (int j = 0; j < maxDoc; j++) {
var expectedHistogram = inputHistograms.get(docBase + j);
if (expectedHistogram == null) {
assertThat(docValues.advanceExact(j), equalTo(false));
expectThrows(IllegalStateException.class, docValues::nextValue);
} else {
assertThat(docValues.advanceExact(j), equalTo(true));
assertThat(docValues.docValueCount(), equalTo(1));
Object actualHistogram = docValues.nextValue();
assertThat(actualHistogram, equalTo(expectedHistogram));
expectThrows(IllegalStateException.class, docValues::nextValue);
}
}
}
}
}
}

@Override
public void testSyntheticSourceKeepArrays() {
// exponential_histogram can't be used within an array
Expand Down
34 changes: 20 additions & 14 deletions ...earch/xpack/exponentialhistogram/aggregations/ExponentialHistogramAggregatorTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import org.apache.lucene.index.IndexableField;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
import org.elasticsearch.exponentialhistogram.ExponentialHistogramTestUtils;
import org.elasticsearch.plugins.SearchPlugin;
Expand All @@ -20,6 +21,7 @@

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.IntStream;
import java.util.stream.Stream;
Expand All @@ -42,25 +44,29 @@ protected static List<ExponentialHistogram> createRandomHistograms(int count) {
return IntStream.range(0, count).mapToObj(i -> ExponentialHistogramTestUtils.randomHistogram()).toList();
}

protected static void addHistogramDoc(
public static void addHistogramDoc(
RandomIndexWriter iw,
String fieldName,
ExponentialHistogram histogram,
@Nullable ExponentialHistogram histogram,
IndexableField... additionalFields
) {
try {
ExponentialHistogramFieldMapper.HistogramDocValueFields docValues = ExponentialHistogramFieldMapper.buildDocValueFields(
fieldName,
histogram.scale(),
IndexWithCount.fromIterator(histogram.negativeBuckets().iterator()),
IndexWithCount.fromIterator(histogram.positiveBuckets().iterator()),
histogram.zeroBucket().zeroThreshold(),
histogram.valueCount(),
histogram.sum(),
histogram.min(),
histogram.max()
);
iw.addDocument(Stream.concat(docValues.fieldsAsList().stream(), Arrays.stream(additionalFields)).toList());
if (histogram == null) {
iw.addDocument(Collections.emptyList());
} else {
ExponentialHistogramFieldMapper.HistogramDocValueFields docValues = ExponentialHistogramFieldMapper.buildDocValueFields(
fieldName,
histogram.scale(),
IndexWithCount.fromIterator(histogram.negativeBuckets().iterator()),
IndexWithCount.fromIterator(histogram.positiveBuckets().iterator()),
histogram.zeroBucket().zeroThreshold(),
histogram.valueCount(),
histogram.sum(),
histogram.min(),
histogram.max()
);
iw.addDocument(Stream.concat(docValues.fieldsAsList().stream(), Arrays.stream(additionalFields)).toList());
}
} catch (IOException e) {
throw new RuntimeException(e);
}
Expand Down