Skip to content

Commit 435e4b2

Browse files
dai-chendai-chen
authored
Add per_minute, per_hour, per_day function support (#4531)
* Support per_minute/hour/day function Signed-off-by: Chen Dai <daichen@amazon.com> * Remove unused UT and IT Signed-off-by: Chen Dai <daichen@amazon.com> * Add more test for edge case Signed-off-by: Chen Dai <daichen@amazon.com> * Address PR comments in doc Signed-off-by: Chen Dai <daichen@amazon.com> * Address PR comments in UT Signed-off-by: Chen Dai <daichen@amazon.com> --------- Signed-off-by: Chen Dai <daichen@amazon.com>
1 parent 5ebed84 commit 435e4b2

File tree

9 files changed

+402
-10
lines changed

9 files changed

+402
-10
lines changed

‎core/src/main/java/org/opensearch/sql/ast/tree/Timechart.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,14 @@ private Timechart timechart(UnresolvedExpression newAggregateFunction) {
123123
return this.toBuilder().aggregateFunction(newAggregateFunction).build();
124124
}
125125

126-
/** TODO: extend to support additional per_* functions */
127126
@RequiredArgsConstructor
128127
static class PerFunction {
129-
private static final Map<String, Integer> UNIT_SECONDS = Map.of("per_second", 1);
128+
private static final Map<String, Integer> UNIT_SECONDS =
129+
Map.of(
130+
"per_second", 1,
131+
"per_minute", 60,
132+
"per_hour", 3600,
133+
"per_day", 86400);
130134
private final String aggName;
131135
private final UnresolvedExpression aggArg;
132136
private final int seconds;

‎core/src/test/java/org/opensearch/sql/ast/tree/TimechartTest.java‎

Lines changed: 88 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@
1414
import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral;
1515
import static org.opensearch.sql.ast.dsl.AstDSL.relation;
1616

17+
import java.util.stream.Stream;
1718
import org.junit.jupiter.api.Test;
1819
import org.junit.jupiter.params.ParameterizedTest;
19-
import org.junit.jupiter.params.provider.CsvSource;
20+
import org.junit.jupiter.params.provider.Arguments;
21+
import org.junit.jupiter.params.provider.MethodSource;
2022
import org.opensearch.sql.ast.dsl.AstDSL;
2123
import org.opensearch.sql.ast.expression.AggregateFunction;
2224
import org.opensearch.sql.ast.expression.Let;
@@ -26,8 +28,23 @@
2628

2729
class TimechartTest {
2830

31+
/**
32+
* @return test sources for per_* function test.
33+
*/
34+
private static Stream<Arguments> perFuncTestSources() {
35+
return Stream.of(
36+
Arguments.of(30, "s", "SECOND"),
37+
Arguments.of(5, "m", "MINUTE"),
38+
Arguments.of(2, "h", "HOUR"),
39+
Arguments.of(1, "d", "DAY"),
40+
Arguments.of(1, "w", "WEEK"),
41+
Arguments.of(1, "M", "MONTH"),
42+
Arguments.of(1, "q", "QUARTER"),
43+
Arguments.of(1, "y", "YEAR"));
44+
}
45+
2946
@ParameterizedTest
30-
@CsvSource({"1, m, MINUTE", "30, s, SECOND", "5, m, MINUTE", "2, h, HOUR", "1, d, DAY"})
47+
@MethodSource("perFuncTestSources")
3148
void should_transform_per_second_for_different_spans(
3249
int spanValue, String spanUnit, String expectedIntervalUnit) {
3350
withTimechart(span(spanValue, spanUnit), perSecond("bytes"))
@@ -45,6 +62,63 @@ void should_transform_per_second_for_different_spans(
4562
timechart(span(spanValue, spanUnit), alias("per_second(bytes)", sum("bytes")))));
4663
}
4764

65+
@ParameterizedTest
66+
@MethodSource("perFuncTestSources")
67+
void should_transform_per_minute_for_different_spans(
68+
int spanValue, String spanUnit, String expectedIntervalUnit) {
69+
withTimechart(span(spanValue, spanUnit), perMinute("bytes"))
70+
.whenTransformingPerFunction()
71+
.thenExpect(
72+
eval(
73+
let(
74+
"per_minute(bytes)",
75+
divide(
76+
multiply("per_minute(bytes)", 60.0),
77+
timestampdiff(
78+
"SECOND",
79+
"@timestamp",
80+
timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))),
81+
timechart(span(spanValue, spanUnit), alias("per_minute(bytes)", sum("bytes")))));
82+
}
83+
84+
@ParameterizedTest
85+
@MethodSource("perFuncTestSources")
86+
void should_transform_per_hour_for_different_spans(
87+
int spanValue, String spanUnit, String expectedIntervalUnit) {
88+
withTimechart(span(spanValue, spanUnit), perHour("bytes"))
89+
.whenTransformingPerFunction()
90+
.thenExpect(
91+
eval(
92+
let(
93+
"per_hour(bytes)",
94+
divide(
95+
multiply("per_hour(bytes)", 3600.0),
96+
timestampdiff(
97+
"SECOND",
98+
"@timestamp",
99+
timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))),
100+
timechart(span(spanValue, spanUnit), alias("per_hour(bytes)", sum("bytes")))));
101+
}
102+
103+
@ParameterizedTest
104+
@MethodSource("perFuncTestSources")
105+
void should_transform_per_day_for_different_spans(
106+
int spanValue, String spanUnit, String expectedIntervalUnit) {
107+
withTimechart(span(spanValue, spanUnit), perDay("bytes"))
108+
.whenTransformingPerFunction()
109+
.thenExpect(
110+
eval(
111+
let(
112+
"per_day(bytes)",
113+
divide(
114+
multiply("per_day(bytes)", 86400.0),
115+
timestampdiff(
116+
"SECOND",
117+
"@timestamp",
118+
timestampadd(expectedIntervalUnit, spanValue, "@timestamp")))),
119+
timechart(span(spanValue, spanUnit), alias("per_day(bytes)", sum("bytes")))));
120+
}
121+
48122
@Test
49123
void should_not_transform_non_per_functions() {
50124
withTimechart(span(1, "m"), sum("bytes"))
@@ -104,6 +178,18 @@ private static AggregateFunction perSecond(String fieldName) {
104178
return (AggregateFunction) aggregate("per_second", field(fieldName));
105179
}
106180

181+
private static AggregateFunction perMinute(String fieldName) {
182+
return (AggregateFunction) aggregate("per_minute", field(fieldName));
183+
}
184+
185+
private static AggregateFunction perHour(String fieldName) {
186+
return (AggregateFunction) aggregate("per_hour", field(fieldName));
187+
}
188+
189+
private static AggregateFunction perDay(String fieldName) {
190+
return (AggregateFunction) aggregate("per_day", field(fieldName));
191+
}
192+
107193
private static AggregateFunction sum(String fieldName) {
108194
return (AggregateFunction) aggregate("sum", field(fieldName));
109195
}

‎docs/user/ppl/cmd/timechart.rst‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,36 @@ Syntax
6969
PER_SECOND
7070
----------
7171

72-
Description
73-
>>>>>>>>>>>
74-
7572
Usage: per_second(field) calculates the per-second rate for a numeric field within each time bucket.
7673

7774
The calculation formula is: `per_second(field) = sum(field) / span_in_seconds`, where `span_in_seconds` is the span interval in seconds.
7875

79-
Note: This function is available since 3.4.0.
76+
Return type: DOUBLE
77+
78+
PER_MINUTE
79+
----------
80+
81+
Usage: per_minute(field) calculates the per-minute rate for a numeric field within each time bucket.
82+
83+
The calculation formula is: `per_minute(field) = sum(field) * 60 / span_in_seconds`, where `span_in_seconds` is the span interval in seconds.
84+
85+
Return type: DOUBLE
86+
87+
PER_HOUR
88+
--------
89+
90+
Usage: per_hour(field) calculates the per-hour rate for a numeric field within each time bucket.
91+
92+
The calculation formula is: `per_hour(field) = sum(field) * 3600 / span_in_seconds`, where `span_in_seconds` is the span interval in seconds.
93+
94+
Return type: DOUBLE
95+
96+
PER_DAY
97+
-------
98+
99+
Usage: per_day(field) calculates the per-day rate for a numeric field within each time bucket.
100+
101+
The calculation formula is: `per_day(field) = sum(field) * 86400 / span_in_seconds`, where `span_in_seconds` is the span interval in seconds.
80102

81103
Return type: DOUBLE
82104

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,36 @@ public void testExplainTimechartPerSecond() throws IOException {
449449
assertTrue(result.contains("per_second(cpu_usage)=[SUM($0)]"));
450450
}
451451

452+
@Test
453+
public void testExplainTimechartPerMinute() throws IOException {
454+
var result = explainQueryToString("source=events | timechart span=2m per_minute(cpu_usage)");
455+
assertTrue(
456+
result.contains(
457+
"per_minute(cpu_usage)=[DIVIDE(*($1, 60.0E0), "
458+
+ "TIMESTAMPDIFF('SECOND':VARCHAR, $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
459+
assertTrue(result.contains("per_minute(cpu_usage)=[SUM($0)]"));
460+
}
461+
462+
@Test
463+
public void testExplainTimechartPerHour() throws IOException {
464+
var result = explainQueryToString("source=events | timechart span=2m per_hour(cpu_usage)");
465+
assertTrue(
466+
result.contains(
467+
"per_hour(cpu_usage)=[DIVIDE(*($1, 3600.0E0), "
468+
+ "TIMESTAMPDIFF('SECOND':VARCHAR, $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
469+
assertTrue(result.contains("per_hour(cpu_usage)=[SUM($0)]"));
470+
}
471+
472+
@Test
473+
public void testExplainTimechartPerDay() throws IOException {
474+
var result = explainQueryToString("source=events | timechart span=2m per_day(cpu_usage)");
475+
assertTrue(
476+
result.contains(
477+
"per_day(cpu_usage)=[DIVIDE(*($1, 86400.0E0), "
478+
+ "TIMESTAMPDIFF('SECOND':VARCHAR, $0, TIMESTAMPADD('MINUTE':VARCHAR, 2, $0)))]"));
479+
assertTrue(result.contains("per_day(cpu_usage)=[SUM($0)]"));
480+
}
481+
452482
@Test
453483
public void noPushDownForAggOnWindow() throws IOException {
454484
enabledOnlyWhenPushdownIsEnabled();

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartPerFunctionIT.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,4 +108,104 @@ public void testTimechartPerSecondWithVariableMonthLengths() throws IOException
108108
rows("2025-02-01 00:00:00", 7.75), // 18748800 / 28 days' seconds
109109
rows("2025-10-01 00:00:00", 7.0)); // 18748800 / 31 days' seconds
110110
}
111+
112+
@Test
113+
public void testTimechartPerMinuteWithSpecifiedSpan() throws IOException {
114+
JSONObject result =
115+
executeQuery(
116+
"source=events_traffic | where month(@timestamp) = 9 | timechart span=2m"
117+
+ " per_minute(packets)");
118+
119+
verifySchema(
120+
result, schema("@timestamp", "timestamp"), schema("per_minute(packets)", "double"));
121+
verifyDataRows(
122+
result,
123+
rows("2025-09-08 10:00:00", 90.0), // (60+120) / 2m
124+
rows("2025-09-08 10:02:00", 120.0)); // (60+180) / 2m
125+
}
126+
127+
@Test
128+
public void testTimechartPerMinuteWithByClause() throws IOException {
129+
JSONObject result =
130+
executeQuery(
131+
"source=events_traffic | where month(@timestamp) = 9 | timechart span=2m"
132+
+ " per_minute(packets) by host");
133+
134+
verifySchema(
135+
result,
136+
schema("@timestamp", "timestamp"),
137+
schema("host", "string"),
138+
schema("per_minute(packets)", "double"));
139+
verifyDataRows(
140+
result,
141+
rows("2025-09-08 10:00:00", "server1", 90.0), // (60+120) / 2m
142+
rows("2025-09-08 10:02:00", "server1", 30.0), // 60 / 2m
143+
rows("2025-09-08 10:02:00", "server2", 90.0)); // 180 / 2m
144+
}
145+
146+
@Test
147+
public void testTimechartPerHourWithSpecifiedSpan() throws IOException {
148+
JSONObject result =
149+
executeQuery(
150+
"source=events_traffic | where month(@timestamp) = 9 | timechart span=2m"
151+
+ " per_hour(packets)");
152+
153+
verifySchema(result, schema("@timestamp", "timestamp"), schema("per_hour(packets)", "double"));
154+
verifyDataRows(
155+
result,
156+
rows("2025-09-08 10:00:00", 5400.0), // (60+120) * 30
157+
rows("2025-09-08 10:02:00", 7200.0)); // (60+180) * 30
158+
}
159+
160+
@Test
161+
public void testTimechartPerHourWithByClause() throws IOException {
162+
JSONObject result =
163+
executeQuery(
164+
"source=events_traffic | where month(@timestamp) = 9 | timechart span=2m"
165+
+ " per_hour(packets) by host");
166+
167+
verifySchema(
168+
result,
169+
schema("@timestamp", "timestamp"),
170+
schema("host", "string"),
171+
schema("per_hour(packets)", "double"));
172+
verifyDataRows(
173+
result,
174+
rows("2025-09-08 10:00:00", "server1", 5400.0), // (60+120) * 30
175+
rows("2025-09-08 10:02:00", "server1", 1800.0), // 60 * 30
176+
rows("2025-09-08 10:02:00", "server2", 5400.0)); // 180 * 30
177+
}
178+
179+
@Test
180+
public void testTimechartPerDayWithSpecifiedSpan() throws IOException {
181+
JSONObject result =
182+
executeQuery(
183+
"source=events_traffic | where month(@timestamp) = 9 | timechart span=2m"
184+
+ " per_day(packets)");
185+
186+
verifySchema(result, schema("@timestamp", "timestamp"), schema("per_day(packets)", "double"));
187+
verifyDataRows(
188+
result,
189+
rows("2025-09-08 10:00:00", 129600.0), // (60+120) * 720
190+
rows("2025-09-08 10:02:00", 172800.0)); // (60+180) * 720
191+
}
192+
193+
@Test
194+
public void testTimechartPerDayWithByClause() throws IOException {
195+
JSONObject result =
196+
executeQuery(
197+
"source=events_traffic | where month(@timestamp) = 9 | timechart span=2m"
198+
+ " per_day(packets) by host");
199+
200+
verifySchema(
201+
result,
202+
schema("@timestamp", "timestamp"),
203+
schema("host", "string"),
204+
schema("per_day(packets)", "double"));
205+
verifyDataRows(
206+
result,
207+
rows("2025-09-08 10:00:00", "server1", 129600.0), // (60+120) * 720
208+
rows("2025-09-08 10:02:00", "server1", 43200.0), // 60 * 720
209+
rows("2025-09-08 10:02:00", "server2", 129600.0)); // 180 * 720
210+
}
111211
}

‎ppl/src/main/antlr/OpenSearchPPLParser.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -711,7 +711,7 @@ percentileApproxFunction
711711
;
712712

713713
perFunction
714-
: funcName=PER_SECOND LT_PRTHS functionArg RT_PRTHS
714+
: funcName=(PER_SECOND | PER_MINUTE | PER_HOUR | PER_DAY) LT_PRTHS functionArg RT_PRTHS
715715
;
716716

717717
numericLiteral

‎ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,24 @@ public void testPerSecondFunctionInTimechartShouldPass() {
105105
assertNotEquals(null, tree);
106106
}
107107

108+
@Test
109+
public void testPerMinuteFunctionInTimechartShouldPass() {
110+
ParseTree tree = new PPLSyntaxParser().parse("source=t | timechart per_minute(a)");
111+
assertNotEquals(null, tree);
112+
}
113+
114+
@Test
115+
public void testPerHourFunctionInTimechartShouldPass() {
116+
ParseTree tree = new PPLSyntaxParser().parse("source=t | timechart per_hour(a)");
117+
assertNotEquals(null, tree);
118+
}
119+
120+
@Test
121+
public void testPerDayFunctionInTimechartShouldPass() {
122+
ParseTree tree = new PPLSyntaxParser().parse("source=t | timechart per_day(a)");
123+
assertNotEquals(null, tree);
124+
}
125+
108126
@Test
109127
public void testDynamicSourceClauseParseTreeStructure() {
110128
String query = "source=[myindex, logs, fieldIndex=\"test\", count=100]";

0 commit comments

Comments
 (0)