diff --git a/NOTICE.TXT b/NOTICE.TXT index 7a5105a33ba..2b90a1f3330 100644 --- a/NOTICE.TXT +++ b/NOTICE.TXT @@ -749,7 +749,7 @@ Notice for: aws-eventstream-1.4.0 limitations under the License. ========== -Notice for: aws-partitions-1.1116.0 +Notice for: aws-partitions-1.1140.0 ---------- @@ -956,7 +956,7 @@ Notice for: aws-partitions-1.1116.0 limitations under the License. ========== -Notice for: aws-sdk-cloudfront-1.119.0 +Notice for: aws-sdk-cloudfront-1.123.0 ---------- @@ -1163,7 +1163,7 @@ Notice for: aws-sdk-cloudfront-1.119.0 limitations under the License. ========== -Notice for: aws-sdk-cloudwatch-1.116.0 +Notice for: aws-sdk-cloudwatch-1.118.0 ---------- @@ -1370,7 +1370,7 @@ Notice for: aws-sdk-cloudwatch-1.116.0 limitations under the License. ========== -Notice for: aws-sdk-core-3.225.2 +Notice for: aws-sdk-core-3.228.0 ---------- @@ -1577,7 +1577,7 @@ Notice for: aws-sdk-core-3.225.2 limitations under the License. ========== -Notice for: aws-sdk-kms-1.105.0 +Notice for: aws-sdk-kms-1.109.0 ---------- @@ -1784,7 +1784,7 @@ Notice for: aws-sdk-kms-1.105.0 limitations under the License. ========== -Notice for: aws-sdk-resourcegroups-1.83.0 +Notice for: aws-sdk-resourcegroups-1.85.0 ---------- @@ -1991,7 +1991,7 @@ Notice for: aws-sdk-resourcegroups-1.83.0 limitations under the License. ========== -Notice for: aws-sdk-s3-1.189.1 +Notice for: aws-sdk-s3-1.195.0 ---------- @@ -2198,7 +2198,7 @@ Notice for: aws-sdk-s3-1.189.1 limitations under the License. ========== -Notice for: aws-sdk-sns-1.100.0 +Notice for: aws-sdk-sns-1.103.0 ---------- @@ -2405,7 +2405,7 @@ Notice for: aws-sdk-sns-1.100.0 limitations under the License. ========== -Notice for: aws-sdk-sqs-1.96.0 +Notice for: aws-sdk-sqs-1.99.0 ---------- @@ -3089,7 +3089,7 @@ limitations under the License. ========== -Notice for: cgi-0.3.6 +Notice for: cgi-0.3.7 ---------- Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved. @@ -3115,7 +3115,7 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: clamp-1.3.2 +Notice for: clamp-1.3.3 ---------- Copyright (c) 2010 Mike Williams @@ -5388,6 +5388,51 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +========== +Notice for: diff-lcs-1.6.2 +---------- + +source: https://github.com/halostatue/diff-lcs/blob/v1.5.0/License.md + +== License + +This software is available under three licenses: the GNU GPL version 2 (or at +your option, a later version), the Perl Artistic license, or the MIT license. +Note that my preference for licensing is the MIT license, but Algorithm::Diff +was dually originally licensed with the Perl Artistic and the GNU GPL ("the +same terms as Perl itself") and given that the Ruby implementation originally +hewed pretty closely to the Perl version, I must maintain the additional +licensing terms. + +* Copyright 2004–2013 Austin Ziegler. +* Adapted from Algorithm::Diff (Perl) by Ned Konz and a Smalltalk version by + Mario I. Wolczko. + +=== MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=== Perl Artistic License (version 2) +See the file docs/artistic.txt in the main distribution. + +=== GNU GPL version 2 +See the file docs/COPYING.txt in the main distribution. ========== Notice for: digest-3.1.0 ---------- @@ -5842,7 +5887,7 @@ source: https://github.com/elastic/elastic-transport-ruby/blob/v8.3.0/LICENSE See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: elasticsearch-8.18.0 +Notice for: elasticsearch-8.19.0 ---------- source: https://github.com/elastic/elasticsearch-ruby/blob/v5.0.4/elasticsearch-api/LICENSE.txt @@ -5862,7 +5907,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: elasticsearch-api-8.18.0 +Notice for: elasticsearch-api-8.19.0 ---------- source: https://github.com/elastic/elasticsearch-ruby/blob/v5.0.4/elasticsearch-transport/LICENSE.txt @@ -5998,7 +6043,7 @@ THE SOFTWARE. Made in Japan ========== -Notice for: faraday-2.13.1 +Notice for: faraday-2.13.4 ---------- source: https://github.com/lostisland/faraday/blob/v0.9.2/LICENSE.md @@ -6012,7 +6057,7 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: faraday-net_http-3.4.0 +Notice for: faraday-net_http-3.4.1 ---------- source: https://github.com/lostisland/faraday-net_http/blob/v1.0.0/LICENSE.md @@ -6976,7 +7021,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: jar-dependencies-0.5.5 +Notice for: jar-dependencies-0.5.4 ---------- source: https://github.com/mkristian/jar-dependencies/blob/0.3.12/MIT-LICENSE @@ -9906,7 +9951,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: jruby-openssl-0.15.4 +Notice for: jruby-openssl-0.15.5 ---------- source: https://github.com/jruby/jruby-openssl/blob/v0.9.21/LICENSE.txt @@ -9993,7 +10038,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: json-2.12.2 +Notice for: json-2.13.2 ---------- source: https://github.com/tmattia/json-generator/blob/v0.1.0/LICENSE.txt @@ -10293,7 +10338,7 @@ source: https://github.com/msgpack/msgpack-ruby/blob/v1.2.4/ext/msgpack/ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: multi_json-1.15.0 +Notice for: multi_json-1.17.0 ---------- source: https://github.com/intridea/multi_json/blob/v1.13.1/LICENSE.md @@ -10402,7 +10447,7 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: mustermann-3.0.3 +Notice for: mustermann-3.0.4 ---------- Copyright (c) 2013-2017 Konstantin Haase @@ -10512,7 +10557,7 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: net-imap-0.5.8 +Notice for: net-imap-0.5.9 ---------- # source: https://github.com/ruby/net-imap/blob/v0.3.7/LICENSE.txt @@ -10726,7 +10771,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: nokogiri-1.18.8 +Notice for: nokogiri-1.18.9 ---------- source: https://github.com/sparklemotion/nokogiri/blob/v1.8.2/LICENSE.md @@ -11071,6 +11116,51 @@ IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +========== +Notice for: org.hdrhistogram:HdrHistogram-2.2.2 +---------- + +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. ========== Notice for: org.javassist:javassist-3.30.2-GA ---------- @@ -11363,7 +11453,7 @@ Eclipse Public License - v 2.0 You may add additional accurate notices of copyright ownership. ========== -Notice for: org.logstash:jvm-options-parser-9.1.0 +Notice for: org.logstash:jvm-options-parser-9.2.0 ---------- Copyright (c) 2022 Elasticsearch B.V. @@ -11686,7 +11776,7 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: puma-6.6.0 +Notice for: puma-6.6.1 ---------- Some code copyright (c) 2005, Zed Shaw @@ -11774,7 +11864,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: rack-3.1.16 +Notice for: rack-3.2.0 ---------- The MIT License (MIT) @@ -12369,6 +12459,99 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +========== +Notice for: rspec-collection_matchers-1.2.1 +---------- + +source: https://github.com/rspec/rspec-collection_matchers/blob/v1.2.1/LICENSE.txt + +(The MIT License) + +Copyright (c) 2013 Hugo Barauna +Copyright (c) 2012 David Chelimsky, Myron Marston +Copyright (c) 2006 David Chelimsky, The RSpec Development Team +Copyright (c) 2005 Steven Baker + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +========== +Notice for: rspec-expectations-3.13.5 +---------- + +source: https://github.com/rspec/rspec-expectations/blob/v3.12.3/LICENSE.md + +The MIT License (MIT) +===================== + +* Copyright © 2012 David Chelimsky, Myron Marston +* Copyright © 2006 David Chelimsky, The RSpec Development Team +* Copyright © 2005 Steven Baker + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +========== +Notice for: rspec-support-3.13.4 +---------- + +source: https://github.com/rspec/rspec-support/blob/v3.12.1/LICENSE.md + +The MIT License (MIT) +==================== + +* Copyright © 2013 David Chelimsky, Myron Marston, Jon Rowe, Sam Phippen, Xavier Shay, Bradley Schaefer + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== Notice for: ruby-progressbar-1.13.0 ---------- @@ -12622,7 +12805,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: sequel-5.93.0 +Notice for: sequel-5.95.0 ---------- Copyright (c) 2007-2008 Sharon Rosner @@ -13715,7 +13898,7 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: tilt-2.6.0 +Notice for: tilt-2.6.1 ---------- Copyright (c) 2010-2016 Ryan Tomayko @@ -14011,6 +14194,34 @@ Copyright (C) 2012 Fluentd Project See the License for the specific language governing permissions and limitations under the License. +========== +Notice for: webrick-1.9.1 +---------- + +source: https://github.com/ruby/webrick/blob/v1.8.1/LICENSE.txt + +Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. ========== Notice for: xml-simple-1.1.9 ---------- diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 14dd90e8199..872d2aa0115 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -245,6 +245,7 @@ dependencies { exclude group: 'com.google.guava', module: 'guava' } implementation 'org.javassist:javassist:3.30.2-GA' + implementation 'org.hdrhistogram:HdrHistogram:2.2.2' testImplementation "org.apache.logging.log4j:log4j-core:${log4jVersion}:tests" testImplementation 'org.hamcrest:hamcrest:2.2' testImplementation 'org.hamcrest:hamcrest-library:2.2' diff --git a/logstash-core/lib/logstash/api/commands/stats.rb b/logstash-core/lib/logstash/api/commands/stats.rb index 5bf1b3e3a09..07842d8d74d 100644 --- a/logstash-core/lib/logstash/api/commands/stats.rb +++ b/logstash-core/lib/logstash/api/commands/stats.rb @@ -174,6 +174,7 @@ def plugin_stats(stats, plugin_type) def report(stats, extended_stats = nil, opts = {}) ret = { + :batch => stats[:batch], :events => stats[:events], :flow => stats[:flow], :plugins => { diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index 81c4516478b..011d471ab2a 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -82,6 +82,7 @@ module Environment Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable, Setting::Password.new("api.ssl.keystore.password", nil, false).nullable, Setting::StringArray.new("api.ssl.supported_protocols", nil, true, %w[TLSv1 TLSv1.1 TLSv1.2 TLSv1.3]), + Setting::SettingString.new("pipeline.batch.metrics", "false", true, ["false", "true"]), Setting::SettingString.new("queue.type", "memory", true, ["persisted", "memory"]), Setting::Boolean.new("queue.drain", false), Setting::Bytes.new("queue.page_capacity", "64mb"), diff --git a/logstash-core/lib/logstash/instrument/metric_type.rb b/logstash-core/lib/logstash/instrument/metric_type.rb index 1f647cf7e48..71393f9c2f7 100644 --- a/logstash-core/lib/logstash/instrument/metric_type.rb +++ b/logstash-core/lib/logstash/instrument/metric_type.rb @@ -29,6 +29,7 @@ def self.create(type, namespaces, key) when :gauge then return org.logstash.instrument.metrics.gauge.LazyDelegatingGauge.new(key.to_s) when :uptime then return org.logstash.instrument.metrics.UptimeMetric.new(key.to_s) when :timer then return org.logstash.instrument.metrics.timer.TimerMetric::create(key.to_s) + when :histogram then return org.logstash.instrument.metrics.histogram.HistogramMetric.new(key.to_s) else fail NameError, "Unknown Metric Type `#{type}`" end end diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index ef90cc1f36b..23775c13ce1 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -58,6 +58,7 @@ def self.included(base) "path.dead_letter_queue", "path.queue", "pipeline.batch.delay", + "pipeline.batch.metrics", "pipeline.batch.size", "pipeline.id", "pipeline.reloadable", diff --git a/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb b/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb index 171525a30a3..427b6a501ca 100644 --- a/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb +++ b/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb @@ -113,7 +113,7 @@ def threaded_read_client end context "WrappedSynchronousQueue" do - let(:queue) { LogStash::WrappedSynchronousQueue.new(1024) } + let(:queue) { LogStash::WrappedSynchronousQueue.new(1024, "false") } before do read_client.set_events_metric(metric.namespace([:stats, :events])) diff --git a/logstash-core/spec/logstash/queue_factory_spec.rb b/logstash-core/spec/logstash/queue_factory_spec.rb index b0aa1a3270a..96b2cf63f50 100644 --- a/logstash-core/spec/logstash/queue_factory_spec.rb +++ b/logstash-core/spec/logstash/queue_factory_spec.rb @@ -31,6 +31,7 @@ LogStash::Setting::SettingNumeric.new("queue.checkpoint.writes", 1024), LogStash::Setting::Boolean.new("queue.checkpoint.retry", false), LogStash::Setting::SettingString.new("pipeline.id", pipeline_id), + LogStash::Setting::SettingString.new("pipeline.batch.metrics", "false", true, ["false", "true"]), LogStash::Setting::SettingPositiveInteger.new("pipeline.batch.size", 125), LogStash::Setting::SettingPositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum) ] diff --git a/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb b/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb index 9e644195806..27c1c019f60 100644 --- a/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb +++ b/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb @@ -19,7 +19,7 @@ require "logstash/instrument/collector" describe LogStash::WrappedSynchronousQueue do - subject {LogStash::WrappedSynchronousQueue.new(5)} + subject {LogStash::WrappedSynchronousQueue.new(5, "false")} describe "queue clients" do context "when requesting a write client" do diff --git a/logstash-core/spec/support/pipeline/pipeline_helpers.rb b/logstash-core/spec/support/pipeline/pipeline_helpers.rb index 2e52d06df55..f6ae12179d4 100644 --- a/logstash-core/spec/support/pipeline/pipeline_helpers.rb +++ b/logstash-core/spec/support/pipeline/pipeline_helpers.rb @@ -73,6 +73,9 @@ def sample_one(sample_event, &block) end describe "\"#{name}\"" do + let(:collector) {LogStash::Instrument::Collector.new} + let(:metric) { LogStash::Instrument::Metric.new(collector).namespace(:null) } + let(:pipeline) do settings.set_value("queue.drain", true) LogStash::JavaPipeline.new( @@ -82,7 +85,7 @@ def sample_one(sample_event, &block) "config_string", "config_string", "input { spec_sampler_input {} }\n" + config + "\noutput { spec_sampler_output {} }" ), settings - ) + ), metric ) end let(:event) do diff --git a/logstash-core/src/main/java/org/logstash/Rubyfier.java b/logstash-core/src/main/java/org/logstash/Rubyfier.java index 4d6288d80b2..0a20da207d1 100644 --- a/logstash-core/src/main/java/org/logstash/Rubyfier.java +++ b/logstash-core/src/main/java/org/logstash/Rubyfier.java @@ -40,6 +40,7 @@ import org.jruby.javasupport.JavaUtil; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.ext.JrubyTimestampExtLibrary; +import org.logstash.instrument.metrics.histogram.HistogramSnapshot; import org.logstash.secret.SecretVariable; public final class Rubyfier { @@ -133,6 +134,7 @@ private static Map, Rubyfier.Converter> initConverters() { ) ); converters.put(SecretVariable.class, JAVAUTIL_CONVERTER); + converters.put(HistogramSnapshot.class, JAVAUTIL_CONVERTER); return converters; } diff --git a/logstash-core/src/main/java/org/logstash/Valuefier.java b/logstash-core/src/main/java/org/logstash/Valuefier.java index 16ebd68a8b7..c2604e4bae5 100644 --- a/logstash-core/src/main/java/org/logstash/Valuefier.java +++ b/logstash-core/src/main/java/org/logstash/Valuefier.java @@ -49,6 +49,7 @@ import org.jruby.javasupport.JavaUtil; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.ext.JrubyTimestampExtLibrary; +import org.logstash.instrument.metrics.histogram.HistogramSnapshot; public final class Valuefier { @@ -201,6 +202,7 @@ RubyUtil.RUBY, new Timestamp(((LocalDateTime) input).toInstant(ZoneOffset.UTC)) RubyUtil.RUBY, new Timestamp(((ZonedDateTime) input).toInstant()) ) ); + converters.put(HistogramSnapshot.class, IDENTITY); return converters; } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java index bcc9b139e58..51232b723cf 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java @@ -68,6 +68,10 @@ public QueueFactoryExt(final Ruby runtime, final RubyClass metaClass) { public static AbstractWrappedQueueExt create(final ThreadContext context, final IRubyObject recv, final IRubyObject settings) throws IOException { final String type = getSetting(context, settings, QUEUE_TYPE_CONTEXT_NAME).asJavaString(); + + final String histogramFlag = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_METRICS) + .asJavaString(); + if (PERSISTED_TYPE.equals(type)) { final Path queuePath = Paths.get( getSetting(context, settings, SettingKeyDefinitions.PATH_QUEUE).asJavaString(), @@ -93,15 +97,14 @@ public static AbstractWrappedQueueExt create(final ThreadContext context, final } ); } else if (MEMORY_TYPE.equals(type)) { + final int batchSize = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) + .convertToInteger().getIntValue(); + final int workers = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_WORKERS) + .convertToInteger().getIntValue(); return new JrubyWrappedSynchronousQueueExt( context.runtime, RubyUtil.WRAPPED_SYNCHRONOUS_QUEUE_CLASS ).initialize( - context, context.runtime.newFixnum( - getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) - .convertToInteger().getIntValue() - * getSetting(context, settings, SettingKeyDefinitions.PIPELINE_WORKERS) - .convertToInteger().getIntValue() - ) + context, context.runtime.newFixnum(batchSize * workers), context.runtime.newString(histogramFlag) ); } else { throw context.runtime.newRaiseException( diff --git a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java index 7c5c47e3162..aee33a3630e 100644 --- a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java +++ b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java @@ -30,6 +30,8 @@ public class SettingKeyDefinitions { public static final String PIPELINE_BATCH_SIZE = "pipeline.batch.size"; + public static final String PIPELINE_BATCH_METRICS = "pipeline.batch.metrics"; + public static final String PATH_QUEUE = "path.queue"; public static final String QUEUE_PAGE_CAPACITY = "queue.page_capacity"; diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index ec26b9b0735..975c8969ed1 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -92,6 +92,7 @@ import org.logstash.instrument.metrics.MetricType; import org.logstash.instrument.metrics.NullMetricExt; import org.logstash.instrument.metrics.UpScaledMetric; +import org.logstash.instrument.metrics.histogram.HistogramMetric; import org.logstash.instrument.metrics.timer.TimerMetric; import org.logstash.instrument.metrics.UptimeMetric; import org.logstash.instrument.metrics.counter.LongCounter; @@ -286,6 +287,12 @@ private AbstractPipelineExt initialize(final ThreadContext context, } catch (InvalidIRException iirex) { throw new IllegalArgumentException(iirex); } + + + // init histogram sample + final RubySymbol[] batchNamespace = buildNamespace(BATCH_KEY); + initOrGetHistogramMetric(context, batchNamespace, BATCH_EVENT_COUNT_KEY); + return this; } @@ -317,8 +324,8 @@ public final IRubyObject openQueue(final ThreadContext context) { new IRubyObject[]{ STATS_KEY, PIPELINES_KEY, - pipelineId.convertToString().intern(), - EVENTS_KEY + pipelineId.convertToString().intern()/*, + EVENTS_KEY*/ } ) ) @@ -640,6 +647,19 @@ private TimerMetric initOrGetTimerMetric(final ThreadContext context, return retrievedMetric.toJava(TimerMetric.class); } + private HistogramMetric initOrGetHistogramMetric(final ThreadContext context, + final RubySymbol[] subPipelineNamespacePath, + final RubySymbol metricName) { + final IRubyObject collector = this.metric.collector(context); + if (collector.isNil()) { + return null; + } + final IRubyObject fullNamespace = pipelineNamespacedPath(subPipelineNamespacePath); + + final IRubyObject retrievedMetric = collector.callMethod(context, "get", new IRubyObject[]{fullNamespace, metricName, context.runtime.newSymbol("histogram")}); + return retrievedMetric.toJava(HistogramMetric.class); + } + private Optional initOrGetNumberGaugeMetric(final ThreadContext context, final RubySymbol[] subPipelineNamespacePath, final RubySymbol metricName) { diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClient.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClient.java index 869a3c758de..2b7f698cbd6 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClient.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClient.java @@ -35,9 +35,9 @@ public interface QueueReadClient { void addFilteredMetrics(int filteredSize); void closeBatch(QueueBatch batch) throws IOException; - public V executeWithTimers(final TimerMetric.ExceptionalSupplier supplier) throws E; + V executeWithTimers(final TimerMetric.ExceptionalSupplier supplier) throws E; - public void executeWithTimers(final TimerMetric.ExceptionalRunnable runnable) throws E; + void executeWithTimers(final TimerMetric.ExceptionalRunnable runnable) throws E; boolean isEmpty(); } diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java index 535cd838a0e..b781a9aeddb 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java @@ -34,6 +34,7 @@ import org.logstash.RubyUtil; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; import org.logstash.instrument.metrics.MetricKeys; +import org.logstash.instrument.metrics.histogram.HistogramMetric; import org.logstash.instrument.metrics.timer.TimerMetric; import org.logstash.instrument.metrics.counter.LongCounter; @@ -41,12 +42,27 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import static org.logstash.instrument.metrics.MetricKeys.BATCH_KEY; +import static org.logstash.instrument.metrics.MetricKeys.EVENTS_KEY; + /** * Common code shared by Persistent and In-Memory queues clients implementation * */ @JRubyClass(name = "QueueReadClientBase") public abstract class QueueReadClientBase extends RubyObject implements QueueReadClient { + public enum BatchSizeSamplingType { + NONE, FULL; + + public static BatchSizeSamplingType decode(String type) { + return switch (type) { + case "false" -> NONE; + case "true" -> FULL; + default -> throw new IllegalArgumentException("Invalid batch size type: " + type); + }; + } + } + private static final long serialVersionUID = 1L; protected int batchSize = 125; @@ -60,6 +76,8 @@ public abstract class QueueReadClientBase extends RubyObject implements QueueRea private transient LongCounter pipelineMetricOut; private transient LongCounter pipelineMetricFiltered; private transient TimerMetric pipelineMetricTime; + private transient HistogramMetric pipelineMetricBatch; + protected BatchSizeSamplingType batchSizeSamplingType = BatchSizeSamplingType.FULL; protected QueueReadClientBase(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); @@ -86,10 +104,14 @@ public IRubyObject setEventsMetric(final IRubyObject metric) { @JRubyMethod(name = "set_pipeline_metric") public IRubyObject setPipelineMetric(final IRubyObject metric) { final AbstractNamespacedMetricExt namespacedMetric = (AbstractNamespacedMetricExt) metric; + ThreadContext context = metric.getRuntime().getCurrentContext(); + AbstractNamespacedMetricExt eventsNamespace = namespacedMetric.namespace(context, EVENTS_KEY); + AbstractNamespacedMetricExt batchNamespace = namespacedMetric.namespace(context, BATCH_KEY); synchronized(namespacedMetric.getMetric()) { - pipelineMetricOut = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.OUT_KEY); - pipelineMetricFiltered = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.FILTERED_KEY); - pipelineMetricTime = TimerMetric.fromRubyBase(namespacedMetric, MetricKeys.DURATION_IN_MILLIS_KEY); + pipelineMetricOut = LongCounter.fromRubyBase(eventsNamespace, MetricKeys.OUT_KEY); + pipelineMetricFiltered = LongCounter.fromRubyBase(eventsNamespace, MetricKeys.FILTERED_KEY); + pipelineMetricTime = TimerMetric.fromRubyBase(eventsNamespace, MetricKeys.DURATION_IN_MILLIS_KEY); + pipelineMetricBatch = HistogramMetric.fromRubyBase(batchNamespace, MetricKeys.BATCH_EVENT_COUNT_KEY); } return this; } @@ -193,6 +215,9 @@ public void startMetrics(QueueBatch batch) { // JTODO getId has been deprecated in JDK 19, when JDK 21 is the target version use threadId() instead long threadId = Thread.currentThread().getId(); inflightBatches.put(threadId, batch); + if (batchSizeSamplingType == BatchSizeSamplingType.FULL) { + pipelineMetricBatch.update(batch.filteredSize()); + } } @Override diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java index 0a93a347c4d..a0e554cf830 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java @@ -47,9 +47,11 @@ public JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass) { @SuppressWarnings("rawtypes") private JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass, - BlockingQueue queue, int batchSize, int waitForMillis) { + BlockingQueue queue, int batchSize, int waitForMillis, + BatchSizeSamplingType batchSizeSamplingType) { super(runtime, metaClass); this.queue = queue; + this.batchSizeSamplingType = batchSizeSamplingType; this.batchSize = batchSize; this.waitForNanos = TimeUnit.NANOSECONDS.convert(waitForMillis, TimeUnit.MILLISECONDS); this.waitForMillis = waitForMillis; @@ -58,8 +60,14 @@ private JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass, @SuppressWarnings("rawtypes") public static JrubyMemoryReadClientExt create(BlockingQueue queue, int batchSize, int waitForMillis) { + return create(queue, batchSize, waitForMillis, BatchSizeSamplingType.FULL); + } + + @SuppressWarnings("rawtypes") + public static JrubyMemoryReadClientExt create(BlockingQueue queue, int batchSize, int waitForMillis, + BatchSizeSamplingType batchSizeSamplingType) { return new JrubyMemoryReadClientExt(RubyUtil.RUBY, - RubyUtil.MEMORY_READ_CLIENT_CLASS, queue, batchSize, waitForMillis); + RubyUtil.MEMORY_READ_CLIENT_CLASS, queue, batchSize, waitForMillis, batchSizeSamplingType); } @Override diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java index dbbfb97de5b..9f311678c20 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java @@ -42,6 +42,7 @@ public final class JrubyWrappedSynchronousQueueExt extends AbstractWrappedQueueE private static final long serialVersionUID = 1L; private transient BlockingQueue queue; + private QueueReadClientBase.BatchSizeSamplingType batchMetricsSamplingType; public JrubyWrappedSynchronousQueueExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); @@ -50,8 +51,11 @@ public JrubyWrappedSynchronousQueueExt(final Ruby runtime, final RubyClass metaC @JRubyMethod @SuppressWarnings("unchecked") public JrubyWrappedSynchronousQueueExt initialize(final ThreadContext context, - IRubyObject size) { + IRubyObject size, + IRubyObject batchMetricsSampling) { int typedSize = ((RubyNumeric)size).getIntValue(); + this.batchMetricsSamplingType = QueueReadClientBase.BatchSizeSamplingType.decode(batchMetricsSampling.asJavaString()); + this.queue = new ArrayBlockingQueue<>(typedSize); return this; } @@ -65,7 +69,7 @@ protected JRubyAbstractQueueWriteClientExt getWriteClient(final ThreadContext co protected QueueReadClientBase getReadClient() { // batch size and timeout are currently hard-coded to 125 and 50ms as values observed // to be reasonable tradeoffs between latency and throughput per PR #8707 - return JrubyMemoryReadClientExt.create(queue, 125, 50); + return JrubyMemoryReadClientExt.create(queue, 125, 50, batchMetricsSamplingType); } @Override diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java index 1a24b3cc03e..936eb8364e3 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java @@ -53,6 +53,11 @@ public IRubyObject timer(final ThreadContext context, final IRubyObject key) { return getTimer(context, key); } + @JRubyMethod + public IRubyObject histogram(final ThreadContext context, final IRubyObject key) { + return getHistogram(context, key); + } + @JRubyMethod(required = 1, optional = 1) public IRubyObject increment(final ThreadContext context, final IRubyObject[] args) { return doIncrement(context, args); @@ -95,6 +100,8 @@ protected abstract IRubyObject getGauge(ThreadContext context, IRubyObject key, protected abstract IRubyObject getTimer(ThreadContext context, IRubyObject key); + protected abstract IRubyObject getHistogram(ThreadContext context, IRubyObject key); + protected abstract IRubyObject doTime(ThreadContext context, IRubyObject key, Block block); protected abstract IRubyObject doReportTime(ThreadContext context, diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java index 758d9309eab..15cbefd9191 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java @@ -60,6 +60,13 @@ public IRubyObject timer(final ThreadContext context, return getTimer(context, namespace, key); } + @JRubyMethod + public IRubyObject histogram(final ThreadContext context, + final IRubyObject namespace, + final IRubyObject key) { + return getHistogram(context, namespace, key); + } + @JRubyMethod(name = "report_time") public IRubyObject reportTime(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final IRubyObject duration) { @@ -83,6 +90,8 @@ protected abstract IRubyObject getGauge(ThreadContext context, IRubyObject names protected abstract IRubyObject getTimer(ThreadContext context, IRubyObject namespace, IRubyObject key); + protected abstract IRubyObject getHistogram(ThreadContext context, IRubyObject namespace, IRubyObject key); + protected abstract IRubyObject doReportTime(ThreadContext context, IRubyObject namespace, IRubyObject key, IRubyObject duration); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java index 1303e1a753a..9f245d9c0d3 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java @@ -54,6 +54,7 @@ public final class MetricExt extends AbstractSimpleMetricExt { private static final RubySymbol GAUGE = RubyUtil.RUBY.newSymbol("gauge"); private static final RubySymbol TIMER = RubyUtil.RUBY.newSymbol("timer"); + private static final RubySymbol HISTOGRAM = RubyUtil.RUBY.newSymbol("histogram"); private static final RubySymbol SET = RubyUtil.RUBY.newSymbol("set"); private static final RubySymbol GET = RubyUtil.RUBY.newSymbol("get"); @@ -153,6 +154,16 @@ protected IRubyObject getTimer(final ThreadContext context, ); } + @Override + protected IRubyObject getHistogram(final ThreadContext context, + final IRubyObject namespace, + final IRubyObject key) { + MetricExt.validateKey(context, null, key); + return collector.callMethod(context, + "get", new IRubyObject[]{normalizeNamespace(namespace), key, HISTOGRAM} + ); + } + @Override protected IRubyObject doReportTime(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final IRubyObject duration) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java index 2730ab83749..fd0c91b0618 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java @@ -116,4 +116,7 @@ private MetricKeys() { public static final RubySymbol WRITES_IN_KEY = RubyUtil.RUBY.newSymbol("writes_in"); + public static final RubySymbol BATCH_EVENT_COUNT_KEY = RubyUtil.RUBY.newSymbol("event_count"); + + public static final RubySymbol BATCH_KEY = RubyUtil.RUBY.newSymbol("batch"); } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java index 0ca15cba310..f78dc63bc25 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java @@ -77,6 +77,11 @@ public enum MetricType { * A flow-rate {@link FlowMetric}, instantiated with one or more backing {@link Metric}{@code }. */ FLOW_RATE("flow/rate"), + + /** + * A histogram metric to record values and obtain percentiles measurements. + * */ + HISTOGRAM_LONG("histogram/long"), ; private final String type; diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java index 8a77b4e2f03..8bbb02f7e0b 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java @@ -84,6 +84,11 @@ protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { return metric.timer(context, namespaceName, key); } + @Override + protected IRubyObject getHistogram(ThreadContext context, IRubyObject key) { + return metric.histogram(context, namespaceName, key); + } + @Override protected IRubyObject doIncrement(final ThreadContext context, final IRubyObject[] args) { if (args.length == 1) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java index c004cc20b88..2e5fbad6cae 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java @@ -94,6 +94,14 @@ protected IRubyObject getTimer(final ThreadContext context, return NULL_TIMER_METRIC; } + @Override + protected IRubyObject getHistogram(final ThreadContext context, + final IRubyObject namespace, + final IRubyObject key) { + MetricExt.validateKey(context, null, key); + return context.nil; + } + @Override protected IRubyObject doReportTime(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final IRubyObject duration) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java index 152541b2271..b92a36023d7 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java @@ -90,6 +90,11 @@ protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { return this.metric.getTimer(context, namespaceName, key); } + @Override + protected IRubyObject getHistogram(ThreadContext context, IRubyObject key) { + return this.metric.getHistogram(context, namespaceName, key); + } + @Override protected IRubyObject doIncrement(final ThreadContext context, final IRubyObject[] args) { return context.nil; diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java new file mode 100644 index 00000000000..42409152121 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramMetric.java @@ -0,0 +1,61 @@ +package org.logstash.instrument.metrics.histogram; + +import org.HdrHistogram.Histogram; +import org.HdrHistogram.Recorder; +import org.jruby.RubySymbol; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; +import org.logstash.instrument.metrics.AbstractMetric; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MetricType; + +public class HistogramMetric extends AbstractMetric { + + private final Recorder histogramRecorder; + + private final Histogram lifetimeHistogram; + + /** + * Constructor + * + * + * @param name The name of this metric. This value may be used for display purposes. + */ + public HistogramMetric(String name) { + super(name); + histogramRecorder = new Recorder(1_000_000, 3); + lifetimeHistogram = histogramRecorder.getIntervalHistogram(); + } + + public static HistogramMetric fromRubyBase(AbstractNamespacedMetricExt metric, RubySymbol key) { + final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); + final IRubyObject histogram = metric.histogram(context, key); + if (!HistogramMetric.class.isAssignableFrom(histogram.getJavaClass())) { + // create dummy histogram + return new HistogramMetric("dummy_histogram"); + } + + return histogram.toJava(HistogramMetric.class); + } + + @Override + public MetricType getType() { + return MetricType.HISTOGRAM_LONG; + } + + @Override + public HistogramSnapshot getValue() { + lifetimeHistogram.add(histogramRecorder.getIntervalHistogram()); + return new HistogramSnapshot(lifetimeHistogram); + } + + /** + * Adds a recorded value. + * + * @param value the value to record + */ + public void update(long value) { + histogramRecorder.recordValue(value); + } +} diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java new file mode 100644 index 00000000000..52240eb6022 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/histogram/HistogramSnapshot.java @@ -0,0 +1,40 @@ +package org.logstash.instrument.metrics.histogram; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.HdrHistogram.Histogram; + +import java.io.Serializable; +import java.util.Map; + +/** + * Class to expose percentiles retrieved from an HdrHistogram. + * */ +public final class HistogramSnapshot implements Serializable { + + private static final long serialVersionUID = 4711735381843512566L; + private final long percentile75; + private final long percentile90; + + public HistogramSnapshot(Histogram hdrHistogram) { + percentile75 = hdrHistogram.getValueAtPercentile(75); + percentile90 = hdrHistogram.getValueAtPercentile(90); + } + + @JsonProperty("p75") + public double get75Percentile() { + return percentile75; + } + + @JsonProperty("p90") + public double get90Percentile() { + return percentile90; + } + + @Override + public String toString() { + return "HistogramSnapshot{" + + "percentile75=" + percentile75 + + ", percentile90=" + percentile90 + + '}'; + } +} diff --git a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java index dff3c4845a3..3e390ceb7f1 100644 --- a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java +++ b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java @@ -26,11 +26,21 @@ import org.jruby.RubyHash; import org.jruby.runtime.ThreadContext; import org.junit.Test; +import org.logstash.Event; import org.logstash.RubyTestBase; +import org.logstash.RubyUtil; import org.logstash.execution.QueueBatch; +import org.logstash.execution.QueueReadClientBase.BatchSizeSamplingType; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MetricKeys; +import org.logstash.instrument.metrics.MockNamespacedMetric; +import org.logstash.instrument.metrics.histogram.HistogramMetric; +import org.logstash.instrument.metrics.histogram.HistogramSnapshot; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.logstash.instrument.metrics.MetricKeys.BATCH_KEY; /** * Tests for {@link JrubyMemoryReadClientExt}. @@ -40,11 +50,14 @@ public final class JrubyMemoryReadClientExtTest extends RubyTestBase { @Test @SuppressWarnings("deprecation") public void testInflightBatchesTracking() throws InterruptedException, IOException { - final BlockingQueue queue = - new ArrayBlockingQueue<>(10); - final JrubyMemoryReadClientExt client = - JrubyMemoryReadClientExt.create(queue, 5, 50); + final BlockingQueue queue = new ArrayBlockingQueue<>(10); + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, BatchSizeSamplingType.NONE); + final ThreadContext context = client.getRuntime().getCurrentContext(); + + AbstractNamespacedMetricExt metric = MockNamespacedMetric.create(); + client.setPipelineMetric(metric); + final QueueBatch batch = client.readBatch(); final RubyHash inflight = client.rubyGetInflightBatches(context); assertThat(inflight.size(), is(1)); @@ -53,4 +66,46 @@ public void testInflightBatchesTracking() throws InterruptedException, IOExcepti client.closeBatch(batch); assertThat(client.rubyGetInflightBatches(context).size(), is(0)); } + + @Test + public void givenNonEmptyQueueAndEnabledBatchSizeMetricThenHistogramContainsData() throws InterruptedException { + final JrubyEventExtLibrary.RubyEvent testEvent = JrubyEventExtLibrary.RubyEvent.newRubyEvent(RubyUtil.RUBY, new Event()); + final BlockingQueue queue = new ArrayBlockingQueue<>(10); + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, BatchSizeSamplingType.FULL); + + AbstractNamespacedMetricExt metric = MockNamespacedMetric.create(); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + + ThreadContext context = metric.getRuntime().getCurrentContext(); + HistogramMetric histogram = HistogramMetric.fromRubyBase(metric.namespace(context, BATCH_KEY), MetricKeys.BATCH_EVENT_COUNT_KEY); + HistogramSnapshot metricSnapshot = histogram.getValue(); + assertEquals(1.0, metricSnapshot.get75Percentile(), 0.0001); + assertEquals(1.0, metricSnapshot.get90Percentile(), 0.0001); + } + + @Test + public void givenNonEmptyQueueAndDisabledBatchSizeMetricThenHistogramIsNotPopulated() throws InterruptedException { + final JrubyEventExtLibrary.RubyEvent testEvent = JrubyEventExtLibrary.RubyEvent.newRubyEvent(RubyUtil.RUBY, new Event()); + final BlockingQueue queue = new ArrayBlockingQueue<>(10); + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, BatchSizeSamplingType.NONE); + + AbstractNamespacedMetricExt metric = MockNamespacedMetric.create(); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + + ThreadContext context = metric.getRuntime().getCurrentContext(); + HistogramMetric histogram = HistogramMetric.fromRubyBase(metric.namespace(context, BATCH_KEY), MetricKeys.BATCH_EVENT_COUNT_KEY); + HistogramSnapshot metricSnapshot = histogram.getValue(); + assertEquals(0.0, metricSnapshot.get75Percentile(), 0.0001); + assertEquals(0.0, metricSnapshot.get90Percentile(), 0.0001); + } } diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java index 21c6a7faf83..1e4308858d4 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java @@ -39,7 +39,7 @@ public class MetricTypeTest { * just duplicates the code, but should cause a developer to think twice if they are changing the public contract. */ @Test - public void ensurePassivity(){ + public void ensurePassivity() { Map nameMap = new HashMap<>(EnumSet.allOf(MetricType.class).size()); nameMap.put(MetricType.COUNTER_LONG, "counter/long"); nameMap.put(MetricType.COUNTER_DECIMAL, "counter/decimal"); @@ -51,6 +51,7 @@ public void ensurePassivity(){ nameMap.put(MetricType.GAUGE_UNKNOWN, "gauge/unknown"); nameMap.put(MetricType.GAUGE_RUBYHASH, "gauge/rubyhash"); nameMap.put(MetricType.GAUGE_RUBYTIMESTAMP, "gauge/rubytimestamp"); + nameMap.put(MetricType.HISTOGRAM_LONG, "histogram/long"); nameMap.put(MetricType.FLOW_RATE, "flow/rate"); //ensure we are testing all of the enumerations diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java new file mode 100644 index 00000000000..276c7bdb4b5 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -0,0 +1,109 @@ +package org.logstash.instrument.metrics; + +import org.jruby.Ruby; +import org.jruby.RubyArray; +import org.jruby.RubyClass; +import org.jruby.RubySymbol; +import org.jruby.runtime.Block; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; +import org.logstash.instrument.metrics.counter.LongCounter; +import org.logstash.instrument.metrics.histogram.HistogramMetric; +import org.logstash.instrument.metrics.timer.TimerMetric; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Trivial implementation of AbstractNamespacedMetricExt where each abstract creation + * metric is implemented by pooling metric instances by name. + * */ +@SuppressWarnings({"rawtypes", "serializable"}) +public class MockNamespacedMetric extends AbstractNamespacedMetricExt { + + private static final long serialVersionUID = -6507123659910450215L; + + private transient final ConcurrentMap metrics = new ConcurrentHashMap<>(); + + public static MockNamespacedMetric create() { + return new MockNamespacedMetric(RubyUtil.RUBY, RubyUtil.NAMESPACED_METRIC_CLASS); + } + + MockNamespacedMetric(final Ruby runtime, final RubyClass metaClass) { + super(runtime, metaClass); + } + + @Override + protected IRubyObject getGauge(ThreadContext context, IRubyObject key, IRubyObject value) { + return null; + } + + @Override + protected RubyArray getNamespaceName(ThreadContext context) { + return null; + } + + @Override + protected IRubyObject getCounter(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), LongCounter::new)); + } + + @Override + protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), TimerMetric::create)); + } + + @Override + protected IRubyObject getHistogram(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), HistogramMetric::new)); + } + + @Override + protected IRubyObject doTime(ThreadContext context, IRubyObject key, Block block) { + return null; + } + + @Override + protected IRubyObject doReportTime(ThreadContext context, IRubyObject key, IRubyObject duration) { + return null; + } + + @Override + protected IRubyObject doIncrement(ThreadContext context, IRubyObject[] args) { + return null; + } + + @Override + protected IRubyObject doDecrement(ThreadContext context, IRubyObject[] args) { + return null; + } + + @Override + public AbstractMetricExt getMetric() { + return NullMetricExt.create(); + } + + @Override + protected AbstractNamespacedMetricExt createNamespaced(ThreadContext context, IRubyObject name) { + return this; + } + + @Override + protected IRubyObject getCollector(ThreadContext context) { + return null; + } + + private static void requireRubySymbol(IRubyObject value, String paramName) { + if (!(value instanceof RubySymbol)) { + throw new IllegalArgumentException(paramName + " must be a RubySymbol instead was: " + value.getClass()); + } + } +} \ No newline at end of file diff --git a/tools/dependencies-report/src/main/resources/licenseMapping.csv b/tools/dependencies-report/src/main/resources/licenseMapping.csv index 128eef59960..86824b4b3d3 100644 --- a/tools/dependencies-report/src/main/resources/licenseMapping.csv +++ b/tools/dependencies-report/src/main/resources/licenseMapping.csv @@ -86,6 +86,7 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL "gems:",https://github.com/rubygems/gems,MIT "gene_pool:",https://github.com/bpardee/gene_pool,MIT "getoptlong:",https://github.com/ruby/getoptlong,BSD-2-Clause +"org.hdrhistogram:HdrHistogram:",https://github.com/HdrHistogram/HdrHistogram,BSD-2-Clause "hitimes:",https://github.com/copiousfreetime/hitimes,ISC "http-cookie:",https://github.com/sparklemotion/http-cookie,MIT "http-form_data:",https://github.com/httprb/form_data,MIT diff --git a/tools/dependencies-report/src/main/resources/notices/org.hdrhistogram!HdrHistogram-NOTICE.txt b/tools/dependencies-report/src/main/resources/notices/org.hdrhistogram!HdrHistogram-NOTICE.txt new file mode 100644 index 00000000000..ce8e7c0ea14 --- /dev/null +++ b/tools/dependencies-report/src/main/resources/notices/org.hdrhistogram!HdrHistogram-NOTICE.txt @@ -0,0 +1,41 @@ +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014, 2015, 2016 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/x-pack/spec/monitoring/schemas/monitoring_document_schema.json b/x-pack/spec/monitoring/schemas/monitoring_document_schema.json index d0c31fd7b19..30f132cb0d0 100644 --- a/x-pack/spec/monitoring/schemas/monitoring_document_schema.json +++ b/x-pack/spec/monitoring/schemas/monitoring_document_schema.json @@ -9,6 +9,26 @@ "successes": { "type": "number" } } }, + "batch": { + "type": "object", + "properties": { + "event_count": { + "type": "object", + "require": [ + "p75", + "p90" + ], + "properties": { + "p75": { + "type": "number" + }, + "p90": { + "type": "number" + } + } + } + } + }, "events": { "type": "object", "required": ["filtered", "in", "duration_in_millis", "out"],