From 5f649dfdf1a9fcc4837daa2f3ed83c86193f5e19 Mon Sep 17 00:00:00 2001 From: andsel Date: Mon, 21 Jul 2025 08:36:20 +0200 Subject: [PATCH 01/17] First idea in adding histogram metric --- logstash-core/build.gradle | 1 + .../lib/logstash/instrument/metric_type.rb | 1 + .../execution/AbstractPipelineExt.java | 17 +++++ .../execution/QueueReadClientBase.java | 4 ++ .../metrics/AbstractNamespacedMetricExt.java | 7 +++ .../metrics/AbstractSimpleMetricExt.java | 9 +++ .../instrument/metrics/MetricExt.java | 11 ++++ .../instrument/metrics/MetricKeys.java | 1 + .../instrument/metrics/MetricType.java | 5 ++ .../metrics/NamespacedMetricExt.java | 5 ++ .../instrument/metrics/NullMetricExt.java | 8 +++ .../metrics/NullNamespacedMetricExt.java | 5 ++ .../metrics/histogram/HistogramMetric.java | 62 +++++++++++++++++++ .../metrics/histogram/HistogramSnapshot.java | 33 ++++++++++ 14 files changed, 169 insertions(+) create mode 100644 logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java create mode 100644 logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 14dd90e8199..872d2aa0115 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -245,6 +245,7 @@ dependencies { exclude group: 'com.google.guava', module: 'guava' } implementation 'org.javassist:javassist:3.30.2-GA' + implementation 'org.hdrhistogram:HdrHistogram:2.2.2' testImplementation "org.apache.logging.log4j:log4j-core:${log4jVersion}:tests" testImplementation 'org.hamcrest:hamcrest:2.2' testImplementation 'org.hamcrest:hamcrest-library:2.2' diff --git a/logstash-core/lib/logstash/instrument/metric_type.rb b/logstash-core/lib/logstash/instrument/metric_type.rb index 1f647cf7e48..71393f9c2f7 100644 --- a/logstash-core/lib/logstash/instrument/metric_type.rb +++ b/logstash-core/lib/logstash/instrument/metric_type.rb @@ -29,6 +29,7 @@ def self.create(type, namespaces, key) when :gauge then return org.logstash.instrument.metrics.gauge.LazyDelegatingGauge.new(key.to_s) when :uptime then return org.logstash.instrument.metrics.UptimeMetric.new(key.to_s) when :timer then return org.logstash.instrument.metrics.timer.TimerMetric::create(key.to_s) + when :histogram then return org.logstash.instrument.metrics.histogram.HistogramMetric.new(key.to_s) else fail NameError, "Unknown Metric Type `#{type}`" end end diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index ec26b9b0735..f5e6b8aa00e 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -92,6 +92,7 @@ import org.logstash.instrument.metrics.MetricType; import org.logstash.instrument.metrics.NullMetricExt; import org.logstash.instrument.metrics.UpScaledMetric; +import org.logstash.instrument.metrics.histogram.HistogramMetric; import org.logstash.instrument.metrics.timer.TimerMetric; import org.logstash.instrument.metrics.UptimeMetric; import org.logstash.instrument.metrics.counter.LongCounter; @@ -286,6 +287,12 @@ private AbstractPipelineExt initialize(final ThreadContext context, } catch (InvalidIRException iirex) { throw new IllegalArgumentException(iirex); } + + + // init histogram sample + final RubySymbol[] eventsNamespace = buildNamespace(EVENTS_KEY); + initOrGetHistogramMetric(context, eventsNamespace, BATCH_SIZE_KEY); + return this; } @@ -640,6 +647,16 @@ private TimerMetric initOrGetTimerMetric(final ThreadContext context, return retrievedMetric.toJava(TimerMetric.class); } + private HistogramMetric initOrGetHistogramMetric(final ThreadContext context, + final RubySymbol[] subPipelineNamespacePath, + final RubySymbol metricName) { + final IRubyObject collector = this.metric.collector(context); + final IRubyObject fullNamespace = pipelineNamespacedPath(subPipelineNamespacePath); + + final IRubyObject retrievedMetric = collector.callMethod(context, "get", new IRubyObject[]{fullNamespace, metricName, context.runtime.newSymbol("histogram")}); + return retrievedMetric.toJava(HistogramMetric.class); + } + private Optional initOrGetNumberGaugeMetric(final ThreadContext context, final RubySymbol[] subPipelineNamespacePath, final RubySymbol metricName) { diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java index 535cd838a0e..b27de409491 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java @@ -34,6 +34,7 @@ import org.logstash.RubyUtil; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; import org.logstash.instrument.metrics.MetricKeys; +import org.logstash.instrument.metrics.histogram.HistogramMetric; import org.logstash.instrument.metrics.timer.TimerMetric; import org.logstash.instrument.metrics.counter.LongCounter; @@ -60,6 +61,7 @@ public abstract class QueueReadClientBase extends RubyObject implements QueueRea private transient LongCounter pipelineMetricOut; private transient LongCounter pipelineMetricFiltered; private transient TimerMetric pipelineMetricTime; + private transient HistogramMetric pipelineMetricBatch; protected QueueReadClientBase(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); @@ -90,6 +92,7 @@ public IRubyObject setPipelineMetric(final IRubyObject metric) { pipelineMetricOut = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.OUT_KEY); pipelineMetricFiltered = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.FILTERED_KEY); pipelineMetricTime = TimerMetric.fromRubyBase(namespacedMetric, MetricKeys.DURATION_IN_MILLIS_KEY); + pipelineMetricBatch = HistogramMetric.fromRubyBase(namespacedMetric, MetricKeys.BATCH_SIZE_KEY); } return this; } @@ -193,6 +196,7 @@ public void startMetrics(QueueBatch batch) { // JTODO getId has been deprecated in JDK 19, when JDK 21 is the target version use threadId() instead long threadId = Thread.currentThread().getId(); inflightBatches.put(threadId, batch); + pipelineMetricBatch.update(batch.filteredSize()); } @Override diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java index 1a24b3cc03e..936eb8364e3 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java @@ -53,6 +53,11 @@ public IRubyObject timer(final ThreadContext context, final IRubyObject key) { return getTimer(context, key); } + @JRubyMethod + public IRubyObject histogram(final ThreadContext context, final IRubyObject key) { + return getHistogram(context, key); + } + @JRubyMethod(required = 1, optional = 1) public IRubyObject increment(final ThreadContext context, final IRubyObject[] args) { return doIncrement(context, args); @@ -95,6 +100,8 @@ protected abstract IRubyObject getGauge(ThreadContext context, IRubyObject key, protected abstract IRubyObject getTimer(ThreadContext context, IRubyObject key); + protected abstract IRubyObject getHistogram(ThreadContext context, IRubyObject key); + protected abstract IRubyObject doTime(ThreadContext context, IRubyObject key, Block block); protected abstract IRubyObject doReportTime(ThreadContext context, diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java index 758d9309eab..15cbefd9191 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java @@ -60,6 +60,13 @@ public IRubyObject timer(final ThreadContext context, return getTimer(context, namespace, key); } + @JRubyMethod + public IRubyObject histogram(final ThreadContext context, + final IRubyObject namespace, + final IRubyObject key) { + return getHistogram(context, namespace, key); + } + @JRubyMethod(name = "report_time") public IRubyObject reportTime(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final IRubyObject duration) { @@ -83,6 +90,8 @@ protected abstract IRubyObject getGauge(ThreadContext context, IRubyObject names protected abstract IRubyObject getTimer(ThreadContext context, IRubyObject namespace, IRubyObject key); + protected abstract IRubyObject getHistogram(ThreadContext context, IRubyObject namespace, IRubyObject key); + protected abstract IRubyObject doReportTime(ThreadContext context, IRubyObject namespace, IRubyObject key, IRubyObject duration); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java index 1303e1a753a..9f245d9c0d3 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java @@ -54,6 +54,7 @@ public final class MetricExt extends AbstractSimpleMetricExt { private static final RubySymbol GAUGE = RubyUtil.RUBY.newSymbol("gauge"); private static final RubySymbol TIMER = RubyUtil.RUBY.newSymbol("timer"); + private static final RubySymbol HISTOGRAM = RubyUtil.RUBY.newSymbol("histogram"); private static final RubySymbol SET = RubyUtil.RUBY.newSymbol("set"); private static final RubySymbol GET = RubyUtil.RUBY.newSymbol("get"); @@ -153,6 +154,16 @@ protected IRubyObject getTimer(final ThreadContext context, ); } + @Override + protected IRubyObject getHistogram(final ThreadContext context, + final IRubyObject namespace, + final IRubyObject key) { + MetricExt.validateKey(context, null, key); + return collector.callMethod(context, + "get", new IRubyObject[]{normalizeNamespace(namespace), key, HISTOGRAM} + ); + } + @Override protected IRubyObject doReportTime(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final IRubyObject duration) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java index 2730ab83749..c3101da2b22 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java @@ -116,4 +116,5 @@ private MetricKeys() { public static final RubySymbol WRITES_IN_KEY = RubyUtil.RUBY.newSymbol("writes_in"); + public static final RubySymbol BATCH_SIZE_KEY = RubyUtil.RUBY.newSymbol("batch_size"); } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java index 0ca15cba310..f78dc63bc25 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java @@ -77,6 +77,11 @@ public enum MetricType { * A flow-rate {@link FlowMetric}, instantiated with one or more backing {@link Metric}{@code }. */ FLOW_RATE("flow/rate"), + + /** + * A histogram metric to record values and obtain percentiles measurements. + * */ + HISTOGRAM_LONG("histogram/long"), ; private final String type; diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java index 8a77b4e2f03..8bbb02f7e0b 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java @@ -84,6 +84,11 @@ protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { return metric.timer(context, namespaceName, key); } + @Override + protected IRubyObject getHistogram(ThreadContext context, IRubyObject key) { + return metric.histogram(context, namespaceName, key); + } + @Override protected IRubyObject doIncrement(final ThreadContext context, final IRubyObject[] args) { if (args.length == 1) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java index c004cc20b88..2e5fbad6cae 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java @@ -94,6 +94,14 @@ protected IRubyObject getTimer(final ThreadContext context, return NULL_TIMER_METRIC; } + @Override + protected IRubyObject getHistogram(final ThreadContext context, + final IRubyObject namespace, + final IRubyObject key) { + MetricExt.validateKey(context, null, key); + return context.nil; + } + @Override protected IRubyObject doReportTime(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final IRubyObject duration) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java index 152541b2271..b92a36023d7 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java @@ -90,6 +90,11 @@ protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { return this.metric.getTimer(context, namespaceName, key); } + @Override + protected IRubyObject getHistogram(ThreadContext context, IRubyObject key) { + return this.metric.getHistogram(context, namespaceName, key); + } + @Override protected IRubyObject doIncrement(final ThreadContext context, final IRubyObject[] args) { return context.nil; diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java new file mode 100644 index 00000000000..743cd3063e4 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java @@ -0,0 +1,62 @@ +package org.logstash.instrument.metrics.histogram; + +import org.HdrHistogram.Histogram; +import org.HdrHistogram.Recorder; +import org.jruby.RubySymbol; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; +import org.logstash.instrument.metrics.AbstractMetric; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MetricType; + +public class HistogramMetric extends AbstractMetric { + + private final Recorder histogramRecorder; + + private final Histogram lifetimeHistogram; + + /** + * Constructor + * + * + * @param name The name of this metric. This value may be used for display purposes. + */ + public HistogramMetric(String name) { + super(name); + histogramRecorder = new Recorder(1_000_000, 3); + lifetimeHistogram = histogramRecorder.getIntervalHistogram(); + } + + public static HistogramMetric fromRubyBase(AbstractNamespacedMetricExt metric, RubySymbol key) { + final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); + final IRubyObject histogram = metric.histogram(context, key); + final HistogramMetric javaTimer; + if (HistogramMetric.class.isAssignableFrom(histogram.getJavaClass())) { + javaTimer = histogram.toJava(HistogramMetric.class); + } else { + throw new IllegalArgumentException("Metric " + key + " is not a histogram"); + } + return javaTimer; + } + + @Override + public MetricType getType() { + return MetricType.HISTOGRAM_LONG; + } + + @Override + public HistogramSnapshot getValue() { + lifetimeHistogram.add(histogramRecorder.getIntervalHistogram()); + return new HistogramSnapshot(lifetimeHistogram); + } + + /** + * Adds a recorded value. + * + * @param value the value to record + */ + public void update(long value) { + histogramRecorder.recordValue(value); + } +} diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java new file mode 100644 index 00000000000..457dfbb5701 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java @@ -0,0 +1,33 @@ +package org.logstash.instrument.metrics.histogram; + +import org.HdrHistogram.Histogram; + +/** + * Class to expose percentiles retrieved from an HdrHistogram. + * */ +public final class HistogramSnapshot { + + private final long percentile75; + private final long percentile90; + + public HistogramSnapshot(Histogram hdrHistogram) { + percentile75 = hdrHistogram.getValueAtPercentile(75); + percentile90 = hdrHistogram.getValueAtPercentile(90); + } + + public double get75Percentile() { + return percentile75; + } + + public double get90Percentile() { + return percentile90; + } + + @Override + public String toString() { + return "HistogramSnapshot{" + + "percentile75=" + percentile75 + + ", percentile90=" + percentile90 + + '}'; + } +} From e51ac4864cc5f18b19ea30b8057745fe07a59603 Mon Sep 17 00:00:00 2001 From: andsel Date: Mon, 4 Aug 2025 12:16:39 +0200 Subject: [PATCH 02/17] Updated licenses after HdrHistogram addition --- NOTICE.TXT | 265 ++++++++++++++++-- .../src/main/resources/licenseMapping.csv | 1 + .../org.hdrhistogram!HdrHistogram-NOTICE.txt | 41 +++ 3 files changed, 280 insertions(+), 27 deletions(-) create mode 100644 tools/dependencies-report/src/main/resources/notices/org.hdrhistogram!HdrHistogram-NOTICE.txt diff --git a/NOTICE.TXT b/NOTICE.TXT index 7a5105a33ba..2b90a1f3330 100644 --- a/NOTICE.TXT +++ b/NOTICE.TXT @@ -749,7 +749,7 @@ Notice for: aws-eventstream-1.4.0 limitations under the License. ========== -Notice for: aws-partitions-1.1116.0 +Notice for: aws-partitions-1.1140.0 ---------- @@ -956,7 +956,7 @@ Notice for: aws-partitions-1.1116.0 limitations under the License. ========== -Notice for: aws-sdk-cloudfront-1.119.0 +Notice for: aws-sdk-cloudfront-1.123.0 ---------- @@ -1163,7 +1163,7 @@ Notice for: aws-sdk-cloudfront-1.119.0 limitations under the License. ========== -Notice for: aws-sdk-cloudwatch-1.116.0 +Notice for: aws-sdk-cloudwatch-1.118.0 ---------- @@ -1370,7 +1370,7 @@ Notice for: aws-sdk-cloudwatch-1.116.0 limitations under the License. ========== -Notice for: aws-sdk-core-3.225.2 +Notice for: aws-sdk-core-3.228.0 ---------- @@ -1577,7 +1577,7 @@ Notice for: aws-sdk-core-3.225.2 limitations under the License. ========== -Notice for: aws-sdk-kms-1.105.0 +Notice for: aws-sdk-kms-1.109.0 ---------- @@ -1784,7 +1784,7 @@ Notice for: aws-sdk-kms-1.105.0 limitations under the License. ========== -Notice for: aws-sdk-resourcegroups-1.83.0 +Notice for: aws-sdk-resourcegroups-1.85.0 ---------- @@ -1991,7 +1991,7 @@ Notice for: aws-sdk-resourcegroups-1.83.0 limitations under the License. ========== -Notice for: aws-sdk-s3-1.189.1 +Notice for: aws-sdk-s3-1.195.0 ---------- @@ -2198,7 +2198,7 @@ Notice for: aws-sdk-s3-1.189.1 limitations under the License. ========== -Notice for: aws-sdk-sns-1.100.0 +Notice for: aws-sdk-sns-1.103.0 ---------- @@ -2405,7 +2405,7 @@ Notice for: aws-sdk-sns-1.100.0 limitations under the License. ========== -Notice for: aws-sdk-sqs-1.96.0 +Notice for: aws-sdk-sqs-1.99.0 ---------- @@ -3089,7 +3089,7 @@ limitations under the License. ========== -Notice for: cgi-0.3.6 +Notice for: cgi-0.3.7 ---------- Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved. @@ -3115,7 +3115,7 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: clamp-1.3.2 +Notice for: clamp-1.3.3 ---------- Copyright (c) 2010 Mike Williams @@ -5388,6 +5388,51 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +========== +Notice for: diff-lcs-1.6.2 +---------- + +source: https://github.com/halostatue/diff-lcs/blob/v1.5.0/License.md + +== License + +This software is available under three licenses: the GNU GPL version 2 (or at +your option, a later version), the Perl Artistic license, or the MIT license. +Note that my preference for licensing is the MIT license, but Algorithm::Diff +was dually originally licensed with the Perl Artistic and the GNU GPL ("the +same terms as Perl itself") and given that the Ruby implementation originally +hewed pretty closely to the Perl version, I must maintain the additional +licensing terms. + +* Copyright 2004–2013 Austin Ziegler. +* Adapted from Algorithm::Diff (Perl) by Ned Konz and a Smalltalk version by + Mario I. Wolczko. + +=== MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=== Perl Artistic License (version 2) +See the file docs/artistic.txt in the main distribution. + +=== GNU GPL version 2 +See the file docs/COPYING.txt in the main distribution. ========== Notice for: digest-3.1.0 ---------- @@ -5842,7 +5887,7 @@ source: https://github.com/elastic/elastic-transport-ruby/blob/v8.3.0/LICENSE See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: elasticsearch-8.18.0 +Notice for: elasticsearch-8.19.0 ---------- source: https://github.com/elastic/elasticsearch-ruby/blob/v5.0.4/elasticsearch-api/LICENSE.txt @@ -5862,7 +5907,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: elasticsearch-api-8.18.0 +Notice for: elasticsearch-api-8.19.0 ---------- source: https://github.com/elastic/elasticsearch-ruby/blob/v5.0.4/elasticsearch-transport/LICENSE.txt @@ -5998,7 +6043,7 @@ THE SOFTWARE. Made in Japan ========== -Notice for: faraday-2.13.1 +Notice for: faraday-2.13.4 ---------- source: https://github.com/lostisland/faraday/blob/v0.9.2/LICENSE.md @@ -6012,7 +6057,7 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: faraday-net_http-3.4.0 +Notice for: faraday-net_http-3.4.1 ---------- source: https://github.com/lostisland/faraday-net_http/blob/v1.0.0/LICENSE.md @@ -6976,7 +7021,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: jar-dependencies-0.5.5 +Notice for: jar-dependencies-0.5.4 ---------- source: https://github.com/mkristian/jar-dependencies/blob/0.3.12/MIT-LICENSE @@ -9906,7 +9951,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: jruby-openssl-0.15.4 +Notice for: jruby-openssl-0.15.5 ---------- source: https://github.com/jruby/jruby-openssl/blob/v0.9.21/LICENSE.txt @@ -9993,7 +10038,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: json-2.12.2 +Notice for: json-2.13.2 ---------- source: https://github.com/tmattia/json-generator/blob/v0.1.0/LICENSE.txt @@ -10293,7 +10338,7 @@ source: https://github.com/msgpack/msgpack-ruby/blob/v1.2.4/ext/msgpack/ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: multi_json-1.15.0 +Notice for: multi_json-1.17.0 ---------- source: https://github.com/intridea/multi_json/blob/v1.13.1/LICENSE.md @@ -10402,7 +10447,7 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: mustermann-3.0.3 +Notice for: mustermann-3.0.4 ---------- Copyright (c) 2013-2017 Konstantin Haase @@ -10512,7 +10557,7 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: net-imap-0.5.8 +Notice for: net-imap-0.5.9 ---------- # source: https://github.com/ruby/net-imap/blob/v0.3.7/LICENSE.txt @@ -10726,7 +10771,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: nokogiri-1.18.8 +Notice for: nokogiri-1.18.9 ---------- source: https://github.com/sparklemotion/nokogiri/blob/v1.8.2/LICENSE.md @@ -11071,6 +11116,51 @@ IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +========== +Notice for: org.hdrhistogram:HdrHistogram-2.2.2 +---------- + +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. ========== Notice for: org.javassist:javassist-3.30.2-GA ---------- @@ -11363,7 +11453,7 @@ Eclipse Public License - v 2.0 You may add additional accurate notices of copyright ownership. ========== -Notice for: org.logstash:jvm-options-parser-9.1.0 +Notice for: org.logstash:jvm-options-parser-9.2.0 ---------- Copyright (c) 2022 Elasticsearch B.V. @@ -11686,7 +11776,7 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: puma-6.6.0 +Notice for: puma-6.6.1 ---------- Some code copyright (c) 2005, Zed Shaw @@ -11774,7 +11864,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: rack-3.1.16 +Notice for: rack-3.2.0 ---------- The MIT License (MIT) @@ -12369,6 +12459,99 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +========== +Notice for: rspec-collection_matchers-1.2.1 +---------- + +source: https://github.com/rspec/rspec-collection_matchers/blob/v1.2.1/LICENSE.txt + +(The MIT License) + +Copyright (c) 2013 Hugo Barauna +Copyright (c) 2012 David Chelimsky, Myron Marston +Copyright (c) 2006 David Chelimsky, The RSpec Development Team +Copyright (c) 2005 Steven Baker + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +========== +Notice for: rspec-expectations-3.13.5 +---------- + +source: https://github.com/rspec/rspec-expectations/blob/v3.12.3/LICENSE.md + +The MIT License (MIT) +===================== + +* Copyright © 2012 David Chelimsky, Myron Marston +* Copyright © 2006 David Chelimsky, The RSpec Development Team +* Copyright © 2005 Steven Baker + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +========== +Notice for: rspec-support-3.13.4 +---------- + +source: https://github.com/rspec/rspec-support/blob/v3.12.1/LICENSE.md + +The MIT License (MIT) +==================== + +* Copyright © 2013 David Chelimsky, Myron Marston, Jon Rowe, Sam Phippen, Xavier Shay, Bradley Schaefer + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== Notice for: ruby-progressbar-1.13.0 ---------- @@ -12622,7 +12805,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: sequel-5.93.0 +Notice for: sequel-5.95.0 ---------- Copyright (c) 2007-2008 Sharon Rosner @@ -13715,7 +13898,7 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: tilt-2.6.0 +Notice for: tilt-2.6.1 ---------- Copyright (c) 2010-2016 Ryan Tomayko @@ -14011,6 +14194,34 @@ Copyright (C) 2012 Fluentd Project See the License for the specific language governing permissions and limitations under the License. +========== +Notice for: webrick-1.9.1 +---------- + +source: https://github.com/ruby/webrick/blob/v1.8.1/LICENSE.txt + +Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. ========== Notice for: xml-simple-1.1.9 ---------- diff --git a/tools/dependencies-report/src/main/resources/licenseMapping.csv b/tools/dependencies-report/src/main/resources/licenseMapping.csv index 128eef59960..86824b4b3d3 100644 --- a/tools/dependencies-report/src/main/resources/licenseMapping.csv +++ b/tools/dependencies-report/src/main/resources/licenseMapping.csv @@ -86,6 +86,7 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL "gems:",https://github.com/rubygems/gems,MIT "gene_pool:",https://github.com/bpardee/gene_pool,MIT "getoptlong:",https://github.com/ruby/getoptlong,BSD-2-Clause +"org.hdrhistogram:HdrHistogram:",https://github.com/HdrHistogram/HdrHistogram,BSD-2-Clause "hitimes:",https://github.com/copiousfreetime/hitimes,ISC "http-cookie:",https://github.com/sparklemotion/http-cookie,MIT "http-form_data:",https://github.com/httprb/form_data,MIT diff --git a/tools/dependencies-report/src/main/resources/notices/org.hdrhistogram!HdrHistogram-NOTICE.txt b/tools/dependencies-report/src/main/resources/notices/org.hdrhistogram!HdrHistogram-NOTICE.txt new file mode 100644 index 00000000000..ce8e7c0ea14 --- /dev/null +++ b/tools/dependencies-report/src/main/resources/notices/org.hdrhistogram!HdrHistogram-NOTICE.txt @@ -0,0 +1,41 @@ +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file From 2b6f4852c46929a3b67e9bf2957466b7f71f2d31 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 6 Aug 2025 12:34:01 +0200 Subject: [PATCH 03/17] Created a test mocking class for namespaced metric to be used to create metrics inside, for example, the queue reader client --- .../ext/JrubyMemoryReadClientExtTest.java | 13 ++- .../instrument/metrics/MetricTypeTest.java | 3 +- .../metrics/MockNamespacedMetric.java | 108 ++++++++++++++++++ 3 files changed, 119 insertions(+), 5 deletions(-) create mode 100644 logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java diff --git a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java index dff3c4845a3..6fc4c10db03 100644 --- a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java +++ b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java @@ -28,6 +28,8 @@ import org.junit.Test; import org.logstash.RubyTestBase; import org.logstash.execution.QueueBatch; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MockNamespacedMetric; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -40,11 +42,14 @@ public final class JrubyMemoryReadClientExtTest extends RubyTestBase { @Test @SuppressWarnings("deprecation") public void testInflightBatchesTracking() throws InterruptedException, IOException { - final BlockingQueue queue = - new ArrayBlockingQueue<>(10); - final JrubyMemoryReadClientExt client = - JrubyMemoryReadClientExt.create(queue, 5, 50); + final BlockingQueue queue = new ArrayBlockingQueue<>(10); + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50); + final ThreadContext context = client.getRuntime().getCurrentContext(); + + AbstractNamespacedMetricExt metric = MockNamespacedMetric.create(); + client.setPipelineMetric(metric); + final QueueBatch batch = client.readBatch(); final RubyHash inflight = client.rubyGetInflightBatches(context); assertThat(inflight.size(), is(1)); diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java index 21c6a7faf83..1e4308858d4 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java @@ -39,7 +39,7 @@ public class MetricTypeTest { * just duplicates the code, but should cause a developer to think twice if they are changing the public contract. */ @Test - public void ensurePassivity(){ + public void ensurePassivity() { Map nameMap = new HashMap<>(EnumSet.allOf(MetricType.class).size()); nameMap.put(MetricType.COUNTER_LONG, "counter/long"); nameMap.put(MetricType.COUNTER_DECIMAL, "counter/decimal"); @@ -51,6 +51,7 @@ public void ensurePassivity(){ nameMap.put(MetricType.GAUGE_UNKNOWN, "gauge/unknown"); nameMap.put(MetricType.GAUGE_RUBYHASH, "gauge/rubyhash"); nameMap.put(MetricType.GAUGE_RUBYTIMESTAMP, "gauge/rubytimestamp"); + nameMap.put(MetricType.HISTOGRAM_LONG, "histogram/long"); nameMap.put(MetricType.FLOW_RATE, "flow/rate"); //ensure we are testing all of the enumerations diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java new file mode 100644 index 00000000000..df06c89c44f --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -0,0 +1,108 @@ +package org.logstash.instrument.metrics; + +import org.jruby.Ruby; +import org.jruby.RubyArray; +import org.jruby.RubyClass; +import org.jruby.RubySymbol; +import org.jruby.runtime.Block; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; +import org.logstash.instrument.metrics.counter.LongCounter; +import org.logstash.instrument.metrics.histogram.HistogramMetric; +import org.logstash.instrument.metrics.timer.TimerMetric; + +import java.util.Objects; + +/** + * Trivial implementation of AbstractNamespacedMetricExt where each abstract creation + * metric is implemented by instantiating a newly fresh metric object. + * */ +@SuppressWarnings({"rawtypes", "serializable"}) +public class MockNamespacedMetric extends AbstractNamespacedMetricExt { + + private static final long serialVersionUID = -6507123659910450215L; + + public static MockNamespacedMetric create() { + return new MockNamespacedMetric(RubyUtil.RUBY, RubyUtil.NAMESPACED_METRIC_CLASS); + } + + MockNamespacedMetric(final Ruby runtime, final RubyClass metaClass) { + super(runtime, metaClass); + } + + @Override + protected IRubyObject getGauge(ThreadContext context, IRubyObject key, IRubyObject value) { + return null; + } + + @Override + protected RubyArray getNamespaceName(ThreadContext context) { + return null; + } + + @Override + protected IRubyObject getCounter(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(new LongCounter(key.asJavaString())); + } + + @Override + protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); +// return RubyUtil.toRubyObject(TimerMetric.create("test_timer")); + return RubyUtil.toRubyObject(TimerMetric.create(key.asJavaString())); + } + + @Override + protected IRubyObject getHistogram(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + +// HistogramMetric metric = new HistogramMetric("test_batch_metric"); + return RubyUtil.toRubyObject(new HistogramMetric(key.asJavaString())); + } + + @Override + protected IRubyObject doTime(ThreadContext context, IRubyObject key, Block block) { + return null; + } + + @Override + protected IRubyObject doReportTime(ThreadContext context, IRubyObject key, IRubyObject duration) { + return null; + } + + @Override + protected IRubyObject doIncrement(ThreadContext context, IRubyObject[] args) { + return null; + } + + @Override + protected IRubyObject doDecrement(ThreadContext context, IRubyObject[] args) { + return null; + } + + @Override + public AbstractMetricExt getMetric() { + return NullMetricExt.create(); + } + + @Override + protected AbstractNamespacedMetricExt createNamespaced(ThreadContext context, IRubyObject name) { + return null; + } + + @Override + protected IRubyObject getCollector(ThreadContext context) { + return null; + } + + private static void requireRubySymbol(IRubyObject value, String paramName) { + if (!(value instanceof RubySymbol)) { + throw new IllegalArgumentException(paramName + " must be a RubySymbol instead was: " + value.getClass()); + } + } +} \ No newline at end of file From 0ba37768a4d78d881205fa536d9342505b27f4e3 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 6 Aug 2025 15:04:38 +0200 Subject: [PATCH 04/17] Fixed test, missed metric passed into JavaPipeline instantiation --- .../spec/support/pipeline/pipeline_helpers.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/logstash-core/spec/support/pipeline/pipeline_helpers.rb b/logstash-core/spec/support/pipeline/pipeline_helpers.rb index 2e52d06df55..4775de26c25 100644 --- a/logstash-core/spec/support/pipeline/pipeline_helpers.rb +++ b/logstash-core/spec/support/pipeline/pipeline_helpers.rb @@ -73,6 +73,14 @@ def sample_one(sample_event, &block) end describe "\"#{name}\"" do + let(:collector) {LogStash::Instrument::Collector.new} + let(:metric) { LogStash::Instrument::Metric.new(collector).namespace(:null) } + # let(:histogram) { LogStash::Instrument::MetricType.create(:histogram, "namespace", "queue_histo") } + # let(:namespaced_metric) { + # mock_metric = double("mock_metric") + # allow(mock_metric).to receive(:get).and_return(histogram) + # } + let(:pipeline) do settings.set_value("queue.drain", true) LogStash::JavaPipeline.new( @@ -82,7 +90,7 @@ def sample_one(sample_event, &block) "config_string", "config_string", "input { spec_sampler_input {} }\n" + config + "\noutput { spec_sampler_output {} }" ), settings - ) + ), metric ) end let(:event) do From 8f884480b86a03841db84e1f2d9b9527240687c0 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 6 Aug 2025 15:40:35 +0200 Subject: [PATCH 05/17] Update initialisation of histogram metric for batch queue reader so that in case no collector is provided it return safely --- .../main/java/org/logstash/execution/AbstractPipelineExt.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index f5e6b8aa00e..9da898c451e 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -651,6 +651,9 @@ private HistogramMetric initOrGetHistogramMetric(final ThreadContext context, final RubySymbol[] subPipelineNamespacePath, final RubySymbol metricName) { final IRubyObject collector = this.metric.collector(context); + if (collector.isNil()) { + return null; + } final IRubyObject fullNamespace = pipelineNamespacedPath(subPipelineNamespacePath); final IRubyObject retrievedMetric = collector.callMethod(context, "get", new IRubyObject[]{fullNamespace, metricName, context.runtime.newSymbol("histogram")}); From 627a9b6c47a2a45d8c5a2c6a4ecaae8352e919c3 Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 7 Aug 2025 15:23:36 +0200 Subject: [PATCH 06/17] If histogram metric is not assignable then create an empty dummy --- .../spec/support/pipeline/pipeline_helpers.rb | 5 ----- .../metrics/histogram/HistogramMetric.java | 17 ++++++++--------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/logstash-core/spec/support/pipeline/pipeline_helpers.rb b/logstash-core/spec/support/pipeline/pipeline_helpers.rb index 4775de26c25..f6ae12179d4 100644 --- a/logstash-core/spec/support/pipeline/pipeline_helpers.rb +++ b/logstash-core/spec/support/pipeline/pipeline_helpers.rb @@ -75,11 +75,6 @@ def sample_one(sample_event, &block) describe "\"#{name}\"" do let(:collector) {LogStash::Instrument::Collector.new} let(:metric) { LogStash::Instrument::Metric.new(collector).namespace(:null) } - # let(:histogram) { LogStash::Instrument::MetricType.create(:histogram, "namespace", "queue_histo") } - # let(:namespaced_metric) { - # mock_metric = double("mock_metric") - # allow(mock_metric).to receive(:get).and_return(histogram) - # } let(:pipeline) do settings.set_value("queue.drain", true) diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java index 743cd3063e4..42409152121 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java @@ -29,15 +29,14 @@ public HistogramMetric(String name) { } public static HistogramMetric fromRubyBase(AbstractNamespacedMetricExt metric, RubySymbol key) { - final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); - final IRubyObject histogram = metric.histogram(context, key); - final HistogramMetric javaTimer; - if (HistogramMetric.class.isAssignableFrom(histogram.getJavaClass())) { - javaTimer = histogram.toJava(HistogramMetric.class); - } else { - throw new IllegalArgumentException("Metric " + key + " is not a histogram"); - } - return javaTimer; + final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); + final IRubyObject histogram = metric.histogram(context, key); + if (!HistogramMetric.class.isAssignableFrom(histogram.getJavaClass())) { + // create dummy histogram + return new HistogramMetric("dummy_histogram"); + } + + return histogram.toJava(HistogramMetric.class); } @Override From 1973503e83567082a3d44ac48e51d5db7ea5bf09 Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 7 Aug 2025 17:18:21 +0200 Subject: [PATCH 07/17] Made HistogramSnapshot to implement Serializable interface to be converted by the Valuefier --- .../src/main/java/org/logstash/Valuefier.java | 2 ++ .../metrics/histogram/HistogramSnapshot.java | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/logstash-core/src/main/java/org/logstash/Valuefier.java b/logstash-core/src/main/java/org/logstash/Valuefier.java index 16ebd68a8b7..c2604e4bae5 100644 --- a/logstash-core/src/main/java/org/logstash/Valuefier.java +++ b/logstash-core/src/main/java/org/logstash/Valuefier.java @@ -49,6 +49,7 @@ import org.jruby.javasupport.JavaUtil; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.ext.JrubyTimestampExtLibrary; +import org.logstash.instrument.metrics.histogram.HistogramSnapshot; public final class Valuefier { @@ -201,6 +202,7 @@ RubyUtil.RUBY, new Timestamp(((LocalDateTime) input).toInstant(ZoneOffset.UTC)) RubyUtil.RUBY, new Timestamp(((ZonedDateTime) input).toInstant()) ) ); + converters.put(HistogramSnapshot.class, IDENTITY); return converters; } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java index 457dfbb5701..898a58e00d2 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java @@ -2,11 +2,15 @@ import org.HdrHistogram.Histogram; +import java.io.Serializable; +import java.util.Map; + /** * Class to expose percentiles retrieved from an HdrHistogram. * */ -public final class HistogramSnapshot { +public final class HistogramSnapshot implements Serializable { + private static final long serialVersionUID = 4711735381843512566L; private final long percentile75; private final long percentile90; @@ -30,4 +34,8 @@ public String toString() { ", percentile90=" + percentile90 + '}'; } +// +// public Map asMap() { +// return null; +// } } From 412a266fe3110cbf4b07ef5314f408ad39abdfaf Mon Sep 17 00:00:00 2001 From: andsel Date: Fri, 8 Aug 2025 14:14:04 +0200 Subject: [PATCH 08/17] Minor, remove commented code --- .../instrument/metrics/histogram/HistogramSnapshot.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java index 898a58e00d2..fd5359e8e71 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java @@ -34,8 +34,4 @@ public String toString() { ", percentile90=" + percentile90 + '}'; } -// -// public Map asMap() { -// return null; -// } } From 4c821496c5e780ea1ecaca7d156916431b3adee3 Mon Sep 17 00:00:00 2001 From: andsel Date: Mon, 11 Aug 2025 11:46:29 +0200 Subject: [PATCH 09/17] Aligned metric docuemnt's JSON schema sent to ES --- .../monitoring/schemas/monitoring_document_schema.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/spec/monitoring/schemas/monitoring_document_schema.json b/x-pack/spec/monitoring/schemas/monitoring_document_schema.json index d0c31fd7b19..d4d6075da7a 100644 --- a/x-pack/spec/monitoring/schemas/monitoring_document_schema.json +++ b/x-pack/spec/monitoring/schemas/monitoring_document_schema.json @@ -16,7 +16,15 @@ "filtered": { "type": "number" }, "in": { "type": "number" }, "duration_in_millis": { "type": "number" }, - "out": { "type": "number" } + "out": { "type": "number" }, + "batch_size": { + "type": "object", + "require": ["75Percentile", "90Percentile"], + "properties": { + "75Percentile": { "type": "number" }, + "90Percentile": { "type": "number" } + } + } } } }, From ce7a920254a084425af95f2ef5f0510333417a39 Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 12 Aug 2025 11:11:20 +0200 Subject: [PATCH 10/17] Updated Rubifier to be able to convert also HistogramSnapshot --- logstash-core/src/main/java/org/logstash/Rubyfier.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/logstash-core/src/main/java/org/logstash/Rubyfier.java b/logstash-core/src/main/java/org/logstash/Rubyfier.java index 4d6288d80b2..0a20da207d1 100644 --- a/logstash-core/src/main/java/org/logstash/Rubyfier.java +++ b/logstash-core/src/main/java/org/logstash/Rubyfier.java @@ -40,6 +40,7 @@ import org.jruby.javasupport.JavaUtil; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.ext.JrubyTimestampExtLibrary; +import org.logstash.instrument.metrics.histogram.HistogramSnapshot; import org.logstash.secret.SecretVariable; public final class Rubyfier { @@ -133,6 +134,7 @@ private static Map, Rubyfier.Converter> initConverters() { ) ); converters.put(SecretVariable.class, JAVAUTIL_CONVERTER); + converters.put(HistogramSnapshot.class, JAVAUTIL_CONVERTER); return converters; } From 8aae756ad4f6202185ddfe4bb7e7b29f0fe3fd58 Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 12 Aug 2025 12:04:27 +0200 Subject: [PATCH 11/17] Minor, removed commented code --- .../org/logstash/instrument/metrics/MockNamespacedMetric.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java index df06c89c44f..b9221e128e4 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -52,7 +52,6 @@ protected IRubyObject getCounter(ThreadContext context, IRubyObject key) { protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { Objects.requireNonNull(key); requireRubySymbol(key, "key"); -// return RubyUtil.toRubyObject(TimerMetric.create("test_timer")); return RubyUtil.toRubyObject(TimerMetric.create(key.asJavaString())); } @@ -60,8 +59,6 @@ protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { protected IRubyObject getHistogram(ThreadContext context, IRubyObject key) { Objects.requireNonNull(key); requireRubySymbol(key, "key"); - -// HistogramMetric metric = new HistogramMetric("test_batch_metric"); return RubyUtil.toRubyObject(new HistogramMetric(key.asJavaString())); } From d01c9aee98173c3746051d3da49c72c7411385c8 Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 12 Aug 2025 17:33:25 +0200 Subject: [PATCH 12/17] Added setting 'pipeline.batch.metrics' to enable/disable the collection of batch size related metrics into histrograms --- logstash-core/lib/logstash/environment.rb | 1 + logstash-core/lib/logstash/settings.rb | 1 + .../instrument/wrapped_write_client_spec.rb | 2 +- logstash-core/spec/logstash/queue_factory_spec.rb | 1 + .../util/wrapped_synchronous_queue_spec.rb | 2 +- .../org/logstash/ackedqueue/QueueFactoryExt.java | 15 +++++++++------ .../logstash/common/SettingKeyDefinitions.java | 2 ++ .../org/logstash/execution/QueueReadClient.java | 4 ++-- .../ext/JrubyWrappedSynchronousQueueExt.java | 8 ++++---- 9 files changed, 22 insertions(+), 14 deletions(-) diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index 81c4516478b..011d471ab2a 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -82,6 +82,7 @@ module Environment Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable, Setting::Password.new("api.ssl.keystore.password", nil, false).nullable, Setting::StringArray.new("api.ssl.supported_protocols", nil, true, %w[TLSv1 TLSv1.1 TLSv1.2 TLSv1.3]), + Setting::SettingString.new("pipeline.batch.metrics", "false", true, ["false", "true"]), Setting::SettingString.new("queue.type", "memory", true, ["persisted", "memory"]), Setting::Boolean.new("queue.drain", false), Setting::Bytes.new("queue.page_capacity", "64mb"), diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index ef90cc1f36b..23775c13ce1 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -58,6 +58,7 @@ def self.included(base) "path.dead_letter_queue", "path.queue", "pipeline.batch.delay", + "pipeline.batch.metrics", "pipeline.batch.size", "pipeline.id", "pipeline.reloadable", diff --git a/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb b/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb index 171525a30a3..427b6a501ca 100644 --- a/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb +++ b/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb @@ -113,7 +113,7 @@ def threaded_read_client end context "WrappedSynchronousQueue" do - let(:queue) { LogStash::WrappedSynchronousQueue.new(1024) } + let(:queue) { LogStash::WrappedSynchronousQueue.new(1024, "false") } before do read_client.set_events_metric(metric.namespace([:stats, :events])) diff --git a/logstash-core/spec/logstash/queue_factory_spec.rb b/logstash-core/spec/logstash/queue_factory_spec.rb index b0aa1a3270a..96b2cf63f50 100644 --- a/logstash-core/spec/logstash/queue_factory_spec.rb +++ b/logstash-core/spec/logstash/queue_factory_spec.rb @@ -31,6 +31,7 @@ LogStash::Setting::SettingNumeric.new("queue.checkpoint.writes", 1024), LogStash::Setting::Boolean.new("queue.checkpoint.retry", false), LogStash::Setting::SettingString.new("pipeline.id", pipeline_id), + LogStash::Setting::SettingString.new("pipeline.batch.metrics", "false", true, ["false", "true"]), LogStash::Setting::SettingPositiveInteger.new("pipeline.batch.size", 125), LogStash::Setting::SettingPositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum) ] diff --git a/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb b/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb index 9e644195806..27c1c019f60 100644 --- a/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb +++ b/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb @@ -19,7 +19,7 @@ require "logstash/instrument/collector" describe LogStash::WrappedSynchronousQueue do - subject {LogStash::WrappedSynchronousQueue.new(5)} + subject {LogStash::WrappedSynchronousQueue.new(5, "false")} describe "queue clients" do context "when requesting a write client" do diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java index bcc9b139e58..51232b723cf 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java @@ -68,6 +68,10 @@ public QueueFactoryExt(final Ruby runtime, final RubyClass metaClass) { public static AbstractWrappedQueueExt create(final ThreadContext context, final IRubyObject recv, final IRubyObject settings) throws IOException { final String type = getSetting(context, settings, QUEUE_TYPE_CONTEXT_NAME).asJavaString(); + + final String histogramFlag = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_METRICS) + .asJavaString(); + if (PERSISTED_TYPE.equals(type)) { final Path queuePath = Paths.get( getSetting(context, settings, SettingKeyDefinitions.PATH_QUEUE).asJavaString(), @@ -93,15 +97,14 @@ public static AbstractWrappedQueueExt create(final ThreadContext context, final } ); } else if (MEMORY_TYPE.equals(type)) { + final int batchSize = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) + .convertToInteger().getIntValue(); + final int workers = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_WORKERS) + .convertToInteger().getIntValue(); return new JrubyWrappedSynchronousQueueExt( context.runtime, RubyUtil.WRAPPED_SYNCHRONOUS_QUEUE_CLASS ).initialize( - context, context.runtime.newFixnum( - getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) - .convertToInteger().getIntValue() - * getSetting(context, settings, SettingKeyDefinitions.PIPELINE_WORKERS) - .convertToInteger().getIntValue() - ) + context, context.runtime.newFixnum(batchSize * workers), context.runtime.newString(histogramFlag) ); } else { throw context.runtime.newRaiseException( diff --git a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java index 7c5c47e3162..aee33a3630e 100644 --- a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java +++ b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java @@ -30,6 +30,8 @@ public class SettingKeyDefinitions { public static final String PIPELINE_BATCH_SIZE = "pipeline.batch.size"; + public static final String PIPELINE_BATCH_METRICS = "pipeline.batch.metrics"; + public static final String PATH_QUEUE = "path.queue"; public static final String QUEUE_PAGE_CAPACITY = "queue.page_capacity"; diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClient.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClient.java index 869a3c758de..2b7f698cbd6 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClient.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClient.java @@ -35,9 +35,9 @@ public interface QueueReadClient { void addFilteredMetrics(int filteredSize); void closeBatch(QueueBatch batch) throws IOException; - public V executeWithTimers(final TimerMetric.ExceptionalSupplier supplier) throws E; + V executeWithTimers(final TimerMetric.ExceptionalSupplier supplier) throws E; - public void executeWithTimers(final TimerMetric.ExceptionalRunnable runnable) throws E; + void executeWithTimers(final TimerMetric.ExceptionalRunnable runnable) throws E; boolean isEmpty(); } diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java index dbbfb97de5b..0ca55b007c9 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java @@ -23,9 +23,7 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; -import org.jruby.Ruby; -import org.jruby.RubyClass; -import org.jruby.RubyNumeric; +import org.jruby.*; import org.jruby.anno.JRubyClass; import org.jruby.anno.JRubyMethod; import org.jruby.runtime.ThreadContext; @@ -50,8 +48,10 @@ public JrubyWrappedSynchronousQueueExt(final Ruby runtime, final RubyClass metaC @JRubyMethod @SuppressWarnings("unchecked") public JrubyWrappedSynchronousQueueExt initialize(final ThreadContext context, - IRubyObject size) { + IRubyObject size, + IRubyObject batchMetricsSampling) { int typedSize = ((RubyNumeric)size).getIntValue(); + String batchMetricsSamplingType = ((RubyString) batchMetricsSampling).asJavaString(); this.queue = new ArrayBlockingQueue<>(typedSize); return this; } From 44047352d07e5816db41ca75b1501cf186ddd0b5 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 13 Aug 2025 09:27:13 +0200 Subject: [PATCH 13/17] [Test] Updated MockNamespacedMetric to pool metrics instances --- .../instrument/metrics/MockNamespacedMetric.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java index b9221e128e4..3c9b5ea7721 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -13,16 +13,20 @@ import org.logstash.instrument.metrics.timer.TimerMetric; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; /** * Trivial implementation of AbstractNamespacedMetricExt where each abstract creation - * metric is implemented by instantiating a newly fresh metric object. + * metric is implemented by pooling metric instances by name. * */ @SuppressWarnings({"rawtypes", "serializable"}) public class MockNamespacedMetric extends AbstractNamespacedMetricExt { private static final long serialVersionUID = -6507123659910450215L; + private transient final ConcurrentMap metrics = new ConcurrentHashMap<>(); + public static MockNamespacedMetric create() { return new MockNamespacedMetric(RubyUtil.RUBY, RubyUtil.NAMESPACED_METRIC_CLASS); } @@ -45,21 +49,21 @@ protected RubyArray getNamespaceName(ThreadContext context) { protected IRubyObject getCounter(ThreadContext context, IRubyObject key) { Objects.requireNonNull(key); requireRubySymbol(key, "key"); - return RubyUtil.toRubyObject(new LongCounter(key.asJavaString())); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), LongCounter::new)); } @Override protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { Objects.requireNonNull(key); requireRubySymbol(key, "key"); - return RubyUtil.toRubyObject(TimerMetric.create(key.asJavaString())); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), TimerMetric::create)); } @Override protected IRubyObject getHistogram(ThreadContext context, IRubyObject key) { Objects.requireNonNull(key); requireRubySymbol(key, "key"); - return RubyUtil.toRubyObject(new HistogramMetric(key.asJavaString())); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), HistogramMetric::new)); } @Override From f34e8b4b726f29a6c8737bec0f8b5563a315c041 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 13 Aug 2025 09:28:01 +0200 Subject: [PATCH 14/17] [Test] Added test to verify that queue client reader updates batch size metrics --- .../ext/JrubyWrappedSynchronousQueueExt.java | 5 +++- .../ext/JrubyMemoryReadClientExtTest.java | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java index 0ca55b007c9..4def199ced0 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java @@ -23,7 +23,10 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; -import org.jruby.*; +import org.jruby.Ruby; +import org.jruby.RubyClass; +import org.jruby.RubyNumeric; +import org.jruby.RubyString; import org.jruby.anno.JRubyClass; import org.jruby.anno.JRubyMethod; import org.jruby.runtime.ThreadContext; diff --git a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java index 6fc4c10db03..02ebe4e798c 100644 --- a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java +++ b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java @@ -26,13 +26,19 @@ import org.jruby.RubyHash; import org.jruby.runtime.ThreadContext; import org.junit.Test; +import org.logstash.Event; import org.logstash.RubyTestBase; +import org.logstash.RubyUtil; import org.logstash.execution.QueueBatch; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MetricKeys; import org.logstash.instrument.metrics.MockNamespacedMetric; +import org.logstash.instrument.metrics.histogram.HistogramMetric; +import org.logstash.instrument.metrics.histogram.HistogramSnapshot; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; /** * Tests for {@link JrubyMemoryReadClientExt}. @@ -58,4 +64,24 @@ public void testInflightBatchesTracking() throws InterruptedException, IOExcepti client.closeBatch(batch); assertThat(client.rubyGetInflightBatches(context).size(), is(0)); } + + @Test + public void givenNonEmptyQueueAndEnabledBatchSizeMetricThenHistogramContainsData() throws InterruptedException { + final JrubyEventExtLibrary.RubyEvent testEvent = JrubyEventExtLibrary.RubyEvent.newRubyEvent(RubyUtil.RUBY, new Event()); + final BlockingQueue queue = new ArrayBlockingQueue<>(10); + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50); + + AbstractNamespacedMetricExt metric = MockNamespacedMetric.create(); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + + HistogramMetric histogram = HistogramMetric.fromRubyBase(metric, MetricKeys.BATCH_SIZE_KEY); + HistogramSnapshot metricSnapshot = histogram.getValue(); + assertEquals(1.0, metricSnapshot.get75Percentile(), 0.0001); + assertEquals(1.0, metricSnapshot.get90Percentile(), 0.0001); + } } From 1e27899c663a3ad32dca02e61b2d80f65b185a16 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 13 Aug 2025 10:14:36 +0200 Subject: [PATCH 15/17] Decoded 'pipeline.batch.metrics' setting into BatchSizeSamplingType and spread around to reach in memory queue client and control the batch size metrics. Covered with tests the readBatch code to verify the effectiveness of the flag --- .../execution/QueueReadClientBase.java | 17 ++++++++++++- .../ext/JrubyMemoryReadClientExt.java | 12 +++++++-- .../ext/JrubyWrappedSynchronousQueueExt.java | 7 +++--- .../ext/JrubyMemoryReadClientExtTest.java | 25 +++++++++++++++++-- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java index b27de409491..efb0b15e790 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java @@ -48,6 +48,18 @@ @JRubyClass(name = "QueueReadClientBase") public abstract class QueueReadClientBase extends RubyObject implements QueueReadClient { + public enum BatchSizeSamplingType { + NONE, FULL; + + public static BatchSizeSamplingType decode(String type) { + return switch (type) { + case "false" -> NONE; + case "true" -> FULL; + default -> throw new IllegalArgumentException("Invalid batch size type: " + type); + }; + } + } + private static final long serialVersionUID = 1L; protected int batchSize = 125; @@ -62,6 +74,7 @@ public abstract class QueueReadClientBase extends RubyObject implements QueueRea private transient LongCounter pipelineMetricFiltered; private transient TimerMetric pipelineMetricTime; private transient HistogramMetric pipelineMetricBatch; + protected BatchSizeSamplingType batchSizeSamplingType = BatchSizeSamplingType.FULL; protected QueueReadClientBase(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); @@ -196,7 +209,9 @@ public void startMetrics(QueueBatch batch) { // JTODO getId has been deprecated in JDK 19, when JDK 21 is the target version use threadId() instead long threadId = Thread.currentThread().getId(); inflightBatches.put(threadId, batch); - pipelineMetricBatch.update(batch.filteredSize()); + if (batchSizeSamplingType == BatchSizeSamplingType.FULL) { + pipelineMetricBatch.update(batch.filteredSize()); + } } @Override diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java index 0a93a347c4d..a0e554cf830 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java @@ -47,9 +47,11 @@ public JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass) { @SuppressWarnings("rawtypes") private JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass, - BlockingQueue queue, int batchSize, int waitForMillis) { + BlockingQueue queue, int batchSize, int waitForMillis, + BatchSizeSamplingType batchSizeSamplingType) { super(runtime, metaClass); this.queue = queue; + this.batchSizeSamplingType = batchSizeSamplingType; this.batchSize = batchSize; this.waitForNanos = TimeUnit.NANOSECONDS.convert(waitForMillis, TimeUnit.MILLISECONDS); this.waitForMillis = waitForMillis; @@ -58,8 +60,14 @@ private JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass, @SuppressWarnings("rawtypes") public static JrubyMemoryReadClientExt create(BlockingQueue queue, int batchSize, int waitForMillis) { + return create(queue, batchSize, waitForMillis, BatchSizeSamplingType.FULL); + } + + @SuppressWarnings("rawtypes") + public static JrubyMemoryReadClientExt create(BlockingQueue queue, int batchSize, int waitForMillis, + BatchSizeSamplingType batchSizeSamplingType) { return new JrubyMemoryReadClientExt(RubyUtil.RUBY, - RubyUtil.MEMORY_READ_CLIENT_CLASS, queue, batchSize, waitForMillis); + RubyUtil.MEMORY_READ_CLIENT_CLASS, queue, batchSize, waitForMillis, batchSizeSamplingType); } @Override diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java index 4def199ced0..9f311678c20 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java @@ -26,7 +26,6 @@ import org.jruby.Ruby; import org.jruby.RubyClass; import org.jruby.RubyNumeric; -import org.jruby.RubyString; import org.jruby.anno.JRubyClass; import org.jruby.anno.JRubyMethod; import org.jruby.runtime.ThreadContext; @@ -43,6 +42,7 @@ public final class JrubyWrappedSynchronousQueueExt extends AbstractWrappedQueueE private static final long serialVersionUID = 1L; private transient BlockingQueue queue; + private QueueReadClientBase.BatchSizeSamplingType batchMetricsSamplingType; public JrubyWrappedSynchronousQueueExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); @@ -54,7 +54,8 @@ public JrubyWrappedSynchronousQueueExt initialize(final ThreadContext context, IRubyObject size, IRubyObject batchMetricsSampling) { int typedSize = ((RubyNumeric)size).getIntValue(); - String batchMetricsSamplingType = ((RubyString) batchMetricsSampling).asJavaString(); + this.batchMetricsSamplingType = QueueReadClientBase.BatchSizeSamplingType.decode(batchMetricsSampling.asJavaString()); + this.queue = new ArrayBlockingQueue<>(typedSize); return this; } @@ -68,7 +69,7 @@ protected JRubyAbstractQueueWriteClientExt getWriteClient(final ThreadContext co protected QueueReadClientBase getReadClient() { // batch size and timeout are currently hard-coded to 125 and 50ms as values observed // to be reasonable tradeoffs between latency and throughput per PR #8707 - return JrubyMemoryReadClientExt.create(queue, 125, 50); + return JrubyMemoryReadClientExt.create(queue, 125, 50, batchMetricsSamplingType); } @Override diff --git a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java index 02ebe4e798c..f81114e669f 100644 --- a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java +++ b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java @@ -30,6 +30,7 @@ import org.logstash.RubyTestBase; import org.logstash.RubyUtil; import org.logstash.execution.QueueBatch; +import org.logstash.execution.QueueReadClientBase.BatchSizeSamplingType; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; import org.logstash.instrument.metrics.MetricKeys; import org.logstash.instrument.metrics.MockNamespacedMetric; @@ -49,7 +50,7 @@ public final class JrubyMemoryReadClientExtTest extends RubyTestBase { @SuppressWarnings("deprecation") public void testInflightBatchesTracking() throws InterruptedException, IOException { final BlockingQueue queue = new ArrayBlockingQueue<>(10); - final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50); + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, BatchSizeSamplingType.NONE); final ThreadContext context = client.getRuntime().getCurrentContext(); @@ -71,7 +72,7 @@ public void givenNonEmptyQueueAndEnabledBatchSizeMetricThenHistogramContainsData final BlockingQueue queue = new ArrayBlockingQueue<>(10); queue.add(testEvent); - final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50); + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, BatchSizeSamplingType.FULL); AbstractNamespacedMetricExt metric = MockNamespacedMetric.create(); client.setPipelineMetric(metric); @@ -84,4 +85,24 @@ public void givenNonEmptyQueueAndEnabledBatchSizeMetricThenHistogramContainsData assertEquals(1.0, metricSnapshot.get75Percentile(), 0.0001); assertEquals(1.0, metricSnapshot.get90Percentile(), 0.0001); } + + @Test + public void givenNonEmptyQueueAndDisabledBatchSizeMetricThenHistogramIsNotPopulated() throws InterruptedException { + final JrubyEventExtLibrary.RubyEvent testEvent = JrubyEventExtLibrary.RubyEvent.newRubyEvent(RubyUtil.RUBY, new Event()); + final BlockingQueue queue = new ArrayBlockingQueue<>(10); + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, BatchSizeSamplingType.NONE); + + AbstractNamespacedMetricExt metric = MockNamespacedMetric.create(); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + + HistogramMetric histogram = HistogramMetric.fromRubyBase(metric, MetricKeys.BATCH_SIZE_KEY); + HistogramSnapshot metricSnapshot = histogram.getValue(); + assertEquals(0.0, metricSnapshot.get75Percentile(), 0.0001); + assertEquals(0.0, metricSnapshot.get90Percentile(), 0.0001); + } } From d02881648d8dc57b53f3b2967d85539deae1cad7 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 13 Aug 2025 18:16:48 +0200 Subject: [PATCH 16/17] Reshaped batch size metric response from {pipeline-name}->events->batch_size->[75Percentile, 90Percentile] to {pipeline_name}->batch->event_count->[p75, p90] --- .../lib/logstash/api/commands/stats.rb | 1 + .../execution/AbstractPipelineExt.java | 8 +++---- .../execution/QueueReadClientBase.java | 14 ++++++++---- .../instrument/metrics/MetricKeys.java | 4 +++- .../metrics/histogram/HistogramSnapshot.java | 3 +++ .../ext/JrubyMemoryReadClientExtTest.java | 7 ++++-- .../metrics/MockNamespacedMetric.java | 2 +- .../schemas/monitoring_document_schema.json | 22 +++++++++++-------- 8 files changed, 40 insertions(+), 21 deletions(-) diff --git a/logstash-core/lib/logstash/api/commands/stats.rb b/logstash-core/lib/logstash/api/commands/stats.rb index 5bf1b3e3a09..07842d8d74d 100644 --- a/logstash-core/lib/logstash/api/commands/stats.rb +++ b/logstash-core/lib/logstash/api/commands/stats.rb @@ -174,6 +174,7 @@ def plugin_stats(stats, plugin_type) def report(stats, extended_stats = nil, opts = {}) ret = { + :batch => stats[:batch], :events => stats[:events], :flow => stats[:flow], :plugins => { diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index 9da898c451e..975c8969ed1 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -290,8 +290,8 @@ private AbstractPipelineExt initialize(final ThreadContext context, // init histogram sample - final RubySymbol[] eventsNamespace = buildNamespace(EVENTS_KEY); - initOrGetHistogramMetric(context, eventsNamespace, BATCH_SIZE_KEY); + final RubySymbol[] batchNamespace = buildNamespace(BATCH_KEY); + initOrGetHistogramMetric(context, batchNamespace, BATCH_EVENT_COUNT_KEY); return this; } @@ -324,8 +324,8 @@ public final IRubyObject openQueue(final ThreadContext context) { new IRubyObject[]{ STATS_KEY, PIPELINES_KEY, - pipelineId.convertToString().intern(), - EVENTS_KEY + pipelineId.convertToString().intern()/*, + EVENTS_KEY*/ } ) ) diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java index efb0b15e790..b781a9aeddb 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java @@ -42,6 +42,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import static org.logstash.instrument.metrics.MetricKeys.BATCH_KEY; +import static org.logstash.instrument.metrics.MetricKeys.EVENTS_KEY; + /** * Common code shared by Persistent and In-Memory queues clients implementation * */ @@ -101,11 +104,14 @@ public IRubyObject setEventsMetric(final IRubyObject metric) { @JRubyMethod(name = "set_pipeline_metric") public IRubyObject setPipelineMetric(final IRubyObject metric) { final AbstractNamespacedMetricExt namespacedMetric = (AbstractNamespacedMetricExt) metric; + ThreadContext context = metric.getRuntime().getCurrentContext(); + AbstractNamespacedMetricExt eventsNamespace = namespacedMetric.namespace(context, EVENTS_KEY); + AbstractNamespacedMetricExt batchNamespace = namespacedMetric.namespace(context, BATCH_KEY); synchronized(namespacedMetric.getMetric()) { - pipelineMetricOut = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.OUT_KEY); - pipelineMetricFiltered = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.FILTERED_KEY); - pipelineMetricTime = TimerMetric.fromRubyBase(namespacedMetric, MetricKeys.DURATION_IN_MILLIS_KEY); - pipelineMetricBatch = HistogramMetric.fromRubyBase(namespacedMetric, MetricKeys.BATCH_SIZE_KEY); + pipelineMetricOut = LongCounter.fromRubyBase(eventsNamespace, MetricKeys.OUT_KEY); + pipelineMetricFiltered = LongCounter.fromRubyBase(eventsNamespace, MetricKeys.FILTERED_KEY); + pipelineMetricTime = TimerMetric.fromRubyBase(eventsNamespace, MetricKeys.DURATION_IN_MILLIS_KEY); + pipelineMetricBatch = HistogramMetric.fromRubyBase(batchNamespace, MetricKeys.BATCH_EVENT_COUNT_KEY); } return this; } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java index c3101da2b22..fd0c91b0618 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java @@ -116,5 +116,7 @@ private MetricKeys() { public static final RubySymbol WRITES_IN_KEY = RubyUtil.RUBY.newSymbol("writes_in"); - public static final RubySymbol BATCH_SIZE_KEY = RubyUtil.RUBY.newSymbol("batch_size"); + public static final RubySymbol BATCH_EVENT_COUNT_KEY = RubyUtil.RUBY.newSymbol("event_count"); + + public static final RubySymbol BATCH_KEY = RubyUtil.RUBY.newSymbol("batch"); } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java index fd5359e8e71..52240eb6022 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java @@ -1,5 +1,6 @@ package org.logstash.instrument.metrics.histogram; +import com.fasterxml.jackson.annotation.JsonProperty; import org.HdrHistogram.Histogram; import java.io.Serializable; @@ -19,10 +20,12 @@ public HistogramSnapshot(Histogram hdrHistogram) { percentile90 = hdrHistogram.getValueAtPercentile(90); } + @JsonProperty("p75") public double get75Percentile() { return percentile75; } + @JsonProperty("p90") public double get90Percentile() { return percentile90; } diff --git a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java index f81114e669f..3e390ceb7f1 100644 --- a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java +++ b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java @@ -40,6 +40,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; +import static org.logstash.instrument.metrics.MetricKeys.BATCH_KEY; /** * Tests for {@link JrubyMemoryReadClientExt}. @@ -80,7 +81,8 @@ public void givenNonEmptyQueueAndEnabledBatchSizeMetricThenHistogramContainsData final QueueBatch batch = client.readBatch(); assertEquals(1, batch.filteredSize()); - HistogramMetric histogram = HistogramMetric.fromRubyBase(metric, MetricKeys.BATCH_SIZE_KEY); + ThreadContext context = metric.getRuntime().getCurrentContext(); + HistogramMetric histogram = HistogramMetric.fromRubyBase(metric.namespace(context, BATCH_KEY), MetricKeys.BATCH_EVENT_COUNT_KEY); HistogramSnapshot metricSnapshot = histogram.getValue(); assertEquals(1.0, metricSnapshot.get75Percentile(), 0.0001); assertEquals(1.0, metricSnapshot.get90Percentile(), 0.0001); @@ -100,7 +102,8 @@ public void givenNonEmptyQueueAndDisabledBatchSizeMetricThenHistogramIsNotPopula final QueueBatch batch = client.readBatch(); assertEquals(1, batch.filteredSize()); - HistogramMetric histogram = HistogramMetric.fromRubyBase(metric, MetricKeys.BATCH_SIZE_KEY); + ThreadContext context = metric.getRuntime().getCurrentContext(); + HistogramMetric histogram = HistogramMetric.fromRubyBase(metric.namespace(context, BATCH_KEY), MetricKeys.BATCH_EVENT_COUNT_KEY); HistogramSnapshot metricSnapshot = histogram.getValue(); assertEquals(0.0, metricSnapshot.get75Percentile(), 0.0001); assertEquals(0.0, metricSnapshot.get90Percentile(), 0.0001); diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java index 3c9b5ea7721..276c7bdb4b5 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -93,7 +93,7 @@ public AbstractMetricExt getMetric() { @Override protected AbstractNamespacedMetricExt createNamespaced(ThreadContext context, IRubyObject name) { - return null; + return this; } @Override diff --git a/x-pack/spec/monitoring/schemas/monitoring_document_schema.json b/x-pack/spec/monitoring/schemas/monitoring_document_schema.json index d4d6075da7a..1363fc06e2d 100644 --- a/x-pack/spec/monitoring/schemas/monitoring_document_schema.json +++ b/x-pack/spec/monitoring/schemas/monitoring_document_schema.json @@ -9,6 +9,18 @@ "successes": { "type": "number" } } }, + "batch": { + "type": "object", + "properties": { + "event_count": { + "type": "object", + "require": ["p75", "p90"], + "properties": { + "p75": { "type": "number" }, + "p90": { "type": "number" } + } + } + }, "events": { "type": "object", "required": ["filtered", "in", "duration_in_millis", "out"], @@ -16,15 +28,7 @@ "filtered": { "type": "number" }, "in": { "type": "number" }, "duration_in_millis": { "type": "number" }, - "out": { "type": "number" }, - "batch_size": { - "type": "object", - "require": ["75Percentile", "90Percentile"], - "properties": { - "75Percentile": { "type": "number" }, - "90Percentile": { "type": "number" } - } - } + "out": { "type": "number" } } } }, From edcf61c7be20fe3b4ca2f8a51ad91cc255532931 Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 14 Aug 2025 08:28:43 +0200 Subject: [PATCH 17/17] [Test] fixed monitoring schema definition, missed a closing curly --- .../schemas/monitoring_document_schema.json | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/spec/monitoring/schemas/monitoring_document_schema.json b/x-pack/spec/monitoring/schemas/monitoring_document_schema.json index 1363fc06e2d..30f132cb0d0 100644 --- a/x-pack/spec/monitoring/schemas/monitoring_document_schema.json +++ b/x-pack/spec/monitoring/schemas/monitoring_document_schema.json @@ -14,12 +14,20 @@ "properties": { "event_count": { "type": "object", - "require": ["p75", "p90"], + "require": [ + "p75", + "p90" + ], "properties": { - "p75": { "type": "number" }, - "p90": { "type": "number" } + "p75": { + "type": "number" + }, + "p90": { + "type": "number" + } } } + } }, "events": { "type": "object",