Skip to content

Commit f2d5e71

Browse files
JonasKunzJonasKunz
authored
ESQL: Implement MIN and MAX agg for exponential histograms (#138091)
1 parent 14c9fb0 commit f2d5e71

File tree

12 files changed

+232
-24
lines changed

12 files changed

+232
-24
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,73 @@ dummy-zero_count_only | "{""scale"":2,""sum"":0.0,""min"":0.0,""max"":0.0,""
1414
dummy-zero_threshold_only | "{""scale"":0,""sum"":0.0,""zero"":{""threshold"":2.0E-5}}"
1515
;
1616

17+
18+
19+
allAggsGrouped
20+
required_capability: exponential_histogram_minmax_support
21+
22+
FROM exp_histo_sample
23+
| EVAL instance = CASE(STARTS_WITH(instance, "dummy"), "dummy-grouped", instance)
24+
| STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75) BY instance
25+
| EVAL p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues
26+
| KEEP instance, min, max, p75
27+
| SORT instance
28+
;
29+
30+
instance:keyword | min:double | max:double | p75:double
31+
dummy-grouped | -100.0 | 50.0 | 8.3457089
32+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
33+
instance-1 | 2.17E-4 | 3.190723 | 0.0016068
34+
instance-2 | 2.2E-4 | 2.744054 | 0.0016068
35+
;
36+
37+
38+
39+
allAggsInlineGrouped
40+
required_capability: exponential_histogram_minmax_support
41+
42+
FROM exp_histo_sample
43+
| INLINE STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75) BY instance
44+
| EVAL p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues
45+
| KEEP instance, min, max, p75
46+
| SORT instance
47+
| Limit 15
48+
;
49+
50+
instance:keyword | min:double | max:double | p75:double
51+
dummy-empty | null | null | null
52+
dummy-full | -100.0 | 50.0 | 10.6666667
53+
dummy-negative_only | -50.0 | -1.0 | -12.8729318
54+
dummy-no_zero_bucket | -100.0 | 50.0 | 10.6666667
55+
dummy-positive_only | 1.0 | 50.0 | 34.7656715
56+
dummy-zero_count_only | 0.0 | 0.0 | 0.0
57+
dummy-zero_threshold_only | null | null | null
58+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
59+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
60+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
61+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
62+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
63+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
64+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
65+
instance-0 | 2.4E-4 | 6.786232 | 0.2608237
66+
;
67+
68+
69+
70+
allAggsOnEmptyHistogram
71+
required_capability: exponential_histogram_minmax_support
72+
73+
FROM exp_histo_sample | WHERE instance == "dummy-empty"
74+
| STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75)
75+
| KEEP min, max, p75
76+
;
77+
78+
min:double | max:double | p75:double
79+
NULL | NULL | NULL
80+
;
81+
82+
83+
1784
ungroupedPercentiles
1885
required_capability: exponential_histogram_percentiles_support
1986

@@ -27,6 +94,8 @@ p0:double | p50:double | p99:double | p100:double
2794
2.17E-4 | 0.0016965 | 0.9472324 | 6.786232
2895
;
2996

97+
98+
3099
groupedPercentiles
31100
required_capability: exponential_histogram_percentiles_support
32101

@@ -42,3 +111,62 @@ instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232
42111
instance-1 | 2.17E-4 | 6.469E-4 | 0.1422151 | 3.190723
43112
instance-2 | 2.2E-4 | 6.469E-4 | 0.0857672 | 2.7059714542564097
44113
;
114+
115+
116+
117+
percentileOnEmptyHistogram
118+
required_capability: exponential_histogram_percentiles_support
119+
120+
FROM exp_histo_sample | WHERE instance == "dummy-empty"
121+
| STATS p50 = PERCENTILE(responseTime,50)
122+
| KEEP p50
123+
;
124+
125+
p50:double
126+
NULL
127+
;
128+
129+
130+
131+
ungroupedMinMax
132+
required_capability: exponential_histogram_minmax_support
133+
134+
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
135+
| STATS min = MIN(responseTime), max = MAX(responseTime)
136+
| KEEP min, max
137+
;
138+
139+
min:double | max:double
140+
2.17E-4 | 6.786232
141+
;
142+
143+
144+
145+
groupedMinMax
146+
required_capability: exponential_histogram_minmax_support
147+
148+
FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy")
149+
| STATS min = MIN(responseTime), max = MAX(responseTime) BY instance
150+
| KEEP instance, min, max
151+
| SORT instance
152+
;
153+
154+
instance:keyword | min:double | max:double
155+
instance-0 | 2.4E-4 | 6.786232
156+
instance-1 | 2.17E-4 | 3.190723
157+
instance-2 | 2.2E-4 | 2.744054
158+
;
159+
160+
161+
162+
minMaxOnEmptyHistogram
163+
required_capability: exponential_histogram_minmax_support
164+
165+
FROM exp_histo_sample | WHERE instance == "dummy-empty"
166+
| STATS min = MIN(responseTime), max = MAX(responseTime)
167+
| KEEP min, max
168+
;
169+
170+
min:double | max:double
171+
NULL | NULL
172+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1562,6 +1562,16 @@ public enum Cap {
15621562
*/
15631563
EXPONENTIAL_HISTOGRAM_TOPN(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG),
15641564

1565+
/**
1566+
* Support for exponential_histogram type in PERCENTILES aggregation.
1567+
*/
1568+
EXPONENTIAL_HISTOGRAM_PERCENTILES_SUPPORT(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG),
1569+
1570+
/**
1571+
* Support for exponential_histogram type in MIN/MAX aggregation.
1572+
*/
1573+
EXPONENTIAL_HISTOGRAM_MINMAX_SUPPORT(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG),
1574+
15651575
/**
15661576
* Create new block when filtering OrdinalBytesRefBlock
15671577
*/
@@ -1652,11 +1662,6 @@ public enum Cap {
16521662

16531663
FULL_TEXT_FUNCTIONS_ACCEPT_NULL_FIELD,
16541664

1655-
/**
1656-
* Support for exponential_histogram type in PERCENTILES aggregation.
1657-
*/
1658-
EXPONENTIAL_HISTOGRAM_PERCENTILES_SUPPORT(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG),
1659-
16601665
/**
16611666
* Support for the temporary work to eventually allow FIRST to work with null and multi-value fields, among other things.
16621667
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.compute.aggregation.MaxIpAggregatorFunctionSupplier;
1818
import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier;
1919
import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder;
20+
import org.elasticsearch.compute.data.ExponentialHistogramBlock;
2021
import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
2122
import org.elasticsearch.xpack.esql.core.expression.Expression;
2223
import org.elasticsearch.xpack.esql.core.expression.Literal;
@@ -30,6 +31,7 @@
3031
import org.elasticsearch.xpack.esql.expression.function.FunctionType;
3132
import org.elasticsearch.xpack.esql.expression.function.Param;
3233
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble;
34+
import org.elasticsearch.xpack.esql.expression.function.scalar.histogram.ExtractHistogramComponent;
3335
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMax;
3436
import org.elasticsearch.xpack.esql.planner.ToAggregator;
3537

@@ -59,7 +61,18 @@ public class Max extends AggregateFunction implements ToAggregator, SurrogateExp
5961
);
6062

6163
@FunctionInfo(
62-
returnType = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "unsigned_long", "version" },
64+
returnType = {
65+
"boolean",
66+
"double",
67+
"integer",
68+
"long",
69+
"date",
70+
"date_nanos",
71+
"ip",
72+
"keyword",
73+
"unsigned_long",
74+
"version",
75+
"exponential_histogram" },
6376
description = "The maximum value of a field.",
6477
type = FunctionType.AGGREGATE,
6578
examples = {
@@ -88,7 +101,8 @@ public Max(
88101
"keyword",
89102
"text",
90103
"unsigned_long",
91-
"version" }
104+
"version",
105+
"exponential_histogram" }
92106
) Expression field
93107
) {
94108
this(source, field, Literal.TRUE, NO_WINDOW);
@@ -126,7 +140,7 @@ public Max replaceChildren(List<Expression> newChildren) {
126140
protected TypeResolution resolveType() {
127141
return TypeResolutions.isType(
128142
field(),
129-
dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE,
143+
dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE || dt == DataType.EXPONENTIAL_HISTOGRAM,
130144
sourceText(),
131145
DEFAULT,
132146
"boolean",
@@ -135,13 +149,14 @@ protected TypeResolution resolveType() {
135149
"string",
136150
"version",
137151
"aggregate_metric_double",
152+
"exponential_histogram",
138153
"numeric except counter types"
139154
);
140155
}
141156

142157
@Override
143158
public DataType dataType() {
144-
if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) {
159+
if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE || field().dataType() == DataType.EXPONENTIAL_HISTOGRAM) {
145160
return DataType.DOUBLE;
146161
}
147162
return field().dataType().noText();
@@ -167,6 +182,14 @@ public Expression surrogate() {
167182
window()
168183
);
169184
}
185+
if (field().dataType() == DataType.EXPONENTIAL_HISTOGRAM) {
186+
return new Max(
187+
source(),
188+
ExtractHistogramComponent.create(source(), field(), ExponentialHistogramBlock.Component.MAX),
189+
filter(),
190+
window()
191+
);
192+
}
170193
return field().foldable() ? new MvMax(source(), field()) : null;
171194
}
172195
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.compute.aggregation.MinIpAggregatorFunctionSupplier;
1818
import org.elasticsearch.compute.aggregation.MinLongAggregatorFunctionSupplier;
1919
import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder;
20+
import org.elasticsearch.compute.data.ExponentialHistogramBlock;
2021
import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
2122
import org.elasticsearch.xpack.esql.core.expression.Expression;
2223
import org.elasticsearch.xpack.esql.core.expression.Literal;
@@ -30,6 +31,7 @@
3031
import org.elasticsearch.xpack.esql.expression.function.FunctionType;
3132
import org.elasticsearch.xpack.esql.expression.function.Param;
3233
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble;
34+
import org.elasticsearch.xpack.esql.expression.function.scalar.histogram.ExtractHistogramComponent;
3335
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
3436
import org.elasticsearch.xpack.esql.planner.ToAggregator;
3537

@@ -88,7 +90,8 @@ public Min(
8890
"keyword",
8991
"text",
9092
"unsigned_long",
91-
"version" }
93+
"version",
94+
"exponential_histogram" }
9295
) Expression field
9396
) {
9497
this(source, field, Literal.TRUE, NO_WINDOW);
@@ -126,7 +129,7 @@ public Min withFilter(Expression filter) {
126129
protected TypeResolution resolveType() {
127130
return TypeResolutions.isType(
128131
field(),
129-
dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE,
132+
dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE || dt == DataType.EXPONENTIAL_HISTOGRAM,
130133
sourceText(),
131134
DEFAULT,
132135
"boolean",
@@ -135,13 +138,14 @@ protected TypeResolution resolveType() {
135138
"string",
136139
"version",
137140
"aggregate_metric_double",
141+
"exponential_histogram",
138142
"numeric except counter types"
139143
);
140144
}
141145

142146
@Override
143147
public DataType dataType() {
144-
if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) {
148+
if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE || field().dataType() == DataType.EXPONENTIAL_HISTOGRAM) {
145149
return DataType.DOUBLE;
146150
}
147151
return field().dataType().noText();
@@ -167,6 +171,14 @@ public Expression surrogate() {
167171
window()
168172
);
169173
}
174+
if (field().dataType() == DataType.EXPONENTIAL_HISTOGRAM) {
175+
return new Min(
176+
source(),
177+
ExtractHistogramComponent.create(source(), field(), ExponentialHistogramBlock.Component.MIN),
178+
filter(),
179+
window()
180+
);
181+
}
170182
return field().foldable() ? new MvMin(source(), field()) : null;
171183
}
172184
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.xpack.esql.core.expression.Expression;
2222
import org.elasticsearch.xpack.esql.core.expression.Expressions;
2323
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
24+
import org.elasticsearch.xpack.esql.core.expression.Literal;
2425
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
2526
import org.elasticsearch.xpack.esql.core.tree.Source;
2627
import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -37,6 +38,7 @@
3738
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable;
3839
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType;
3940
import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
41+
import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
4042
import static org.elasticsearch.xpack.esql.core.type.DataType.LONG;
4143

4244
/**
@@ -78,6 +80,10 @@ private ExtractHistogramComponent(StreamInput in) throws IOException {
7880
this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class));
7981
}
8082

83+
public static Expression create(Source source, Expression field, ExponentialHistogramBlock.Component component) {
84+
return new ExtractHistogramComponent(source, field, new Literal(source, component.ordinal(), INTEGER));
85+
}
86+
8187
Expression field() {
8288
return field;
8389
}

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,7 +1152,8 @@ public void testAggregateOnCounter() {
11521152
error("FROM test | STATS min(network.bytes_in)", tsdb),
11531153
equalTo(
11541154
"1:19: argument of [min(network.bytes_in)] must be"
1155-
+ " [boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types],"
1155+
+ " [boolean, date, ip, string, version, aggregate_metric_double,"
1156+
+ " exponential_histogram or numeric except counter types],"
11561157
+ " found value [network.bytes_in] type [counter_long]"
11571158
)
11581159
);
@@ -1161,7 +1162,8 @@ public void testAggregateOnCounter() {
11611162
error("FROM test | STATS max(network.bytes_in)", tsdb),
11621163
equalTo(
11631164
"1:19: argument of [max(network.bytes_in)] must be"
1164-
+ " [boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types],"
1165+
+ " [boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram"
1166+
+ " or numeric except counter types],"
11651167
+ " found value [network.bytes_in] type [counter_long]"
11661168
)
11671169
);

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerP
3737
false,
3838
validPerPosition,
3939
signature,
40-
(v, p) -> "boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types"
40+
(v, p) -> "boolean, date, ip, string, version, aggregate_metric_double, "
41+
+ "exponential_histogram or numeric except counter types"
4142
)
4243
);
4344
}

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxOverTimeErrorTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ protected Matcher<String> expectedTypeErrorMatcher(List<Set<DataType>> validPerP
3737
false,
3838
validPerPosition,
3939
signature,
40-
(v, p) -> "boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types"
40+
(v, p) -> "boolean, date, ip, string, version, aggregate_metric_double, "
41+
+ "exponential_histogram or numeric except counter types"
4142
)
4243
);
4344
}

0 commit comments

Comments
 (0)