From 862768ac1b99dc419ddb72a0ddb10200777f6821 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 22 Oct 2025 17:09:35 -0700 Subject: [PATCH 01/11] Remove duplicate gems when producting logstash artifacts Bundler is used to manage a gem environment that is shipped with logstash artifacts. By default, bundler will install newer/duplicate gems than shipped with ruby distributions (in logstash's case jruby). Duplicate gems in the shipped environment can cause issues with code loading with ambiguous gem specs or gem activation issues. This commit adds a step to compute the duplicate gems managed with bundler (and therefore direct/transitive dependencies of logstash/plugins) and *removes* copies shipped with jruby. Note that there are two locations to do the deduplication at. Both the stdlib gems as well as what jruby refers to as "bundled" gems. The existing pattern for excluding files from artifacts is used to implement the deduplication. --- rakelib/artifacts.rake | 65 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 2739e1fc6a7..b31ee660633 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -82,6 +82,56 @@ namespace "artifact" do res end + def duplicated_gems_exclude_paths + shared_gems_path = 'vendor/jruby/lib/ruby/gems/shared/gems' + default_gemspecs_path = 'vendor/jruby/lib/ruby/gems/shared/specifications/default' + bundle_gems_path = 'vendor/bundle/jruby/*/gems' + + exclusions = [] + + # "bundled" gems in jruby + # https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L108-L152 + shared_gem_names = Dir.glob(File.join(shared_gems_path, '*')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + # "default" gems in jruby/ruby + # https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L21-L106 + default_gem_names = Dir.glob(File.join(default_gemspecs_path, '*.gemspec')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + # gems we explicitly manage with bundler (we always want these to take precedence) + bundle_gem_names = Dir.glob(File.join(bundle_gems_path, '*')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + shared_duplicates = shared_gem_names & bundle_gem_names + default_duplicates = default_gem_names & bundle_gem_names + all_duplicates = (shared_duplicates + default_duplicates).uniq + puts "Adding duplicate gems to exclude path: #{all_duplicates.sort.join(', ')}" + + # Exclude shared/bundled gems duplicates + shared_duplicates.each do |gem_name| + exclusions << "vendor/jruby/lib/ruby/gems/shared/gems/#{gem_name}-*/**/*" + exclusions << "vendor/jruby/lib/ruby/gems/shared/gems/#{gem_name}-*" + exclusions << "vendor/jruby/lib/ruby/gems/shared/specifications/#{gem_name}-*.gemspec" + end + + # Exclude default gems duplicates + default_duplicates.each do |gem_name| + exclusions << "vendor/jruby/lib/ruby/gems/shared/specifications/default/#{gem_name}-*.gemspec" + exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}.rb" + exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}/**/*" + exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}" + end + + exclusions + end + def default_exclude_paths return @exclude_paths if @exclude_paths @@ -101,18 +151,9 @@ namespace "artifact" do @exclude_paths << 'vendor/**/gems/**/Gemfile.lock' @exclude_paths << 'vendor/**/gems/**/Gemfile' - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/gems/rake-*' - # exclude ruby-maven-libs 3.3.9 jars until JRuby ships with >= 3.8.9 - @exclude_paths << 'vendor/bundle/jruby/**/gems/ruby-maven-libs-3.3.9/**/*' - - # remove this after JRuby includes rexml 3.3.x - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/gems/rexml-3.2.5/**/*' - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/specifications/rexml-3.2.5.gemspec' - - # remove this after JRuby includes net-imap-0.2.4+ - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/specifications/net-imap-0.2.3.gemspec' - @exclude_paths << 'vendor/jruby/lib/ruby/gems/shared/gems/net-imap-0.2.3/**/*' - + @exclude_paths.concat(duplicated_gems_exclude_paths) + puts "Full exclude_paths list:" + @exclude_paths.each { |path| puts " - #{path}" } @exclude_paths.freeze end From 2e588a14f3da95c432d12aafcd88beca11500245 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Mon, 27 Oct 2025 14:58:51 -0700 Subject: [PATCH 02/11] only remove gemspecs for duplicated stdlib gems --- rakelib/artifacts.rake | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index b31ee660633..1856b39e6ae 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -123,10 +123,12 @@ namespace "artifact" do # Exclude default gems duplicates default_duplicates.each do |gem_name| + # CODEREVIEW: removing the code itself causes issues with gem loading. Remove only the gemspecs + # for duplicated gems. This should help the code scanning case. exclusions << "vendor/jruby/lib/ruby/gems/shared/specifications/default/#{gem_name}-*.gemspec" - exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}.rb" - exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}/**/*" - exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}" + # exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}.rb" + # exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}/**/*" + # exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}" end exclusions From 4c901beddbbeffa87bbb3d5dffa7138588f6947b Mon Sep 17 00:00:00 2001 From: donoghuc Date: Mon, 3 Nov 2025 14:13:07 -0800 Subject: [PATCH 03/11] Make deduplicate a separate rake task and prevent gradle errors Deduplication should happen as a depenedency of installing default gems. In the current workflow we have a top level gradle task for packaging which calls out to rake. Rake then invokes a *separate* gradle process. When we modify the jruby default, when the separate gradle process goes to check of jruby is installed, it sees a modified jruby and tries to re-install. We work around this by changing how gradle detects if jruby is required to be installed. --- rakelib/artifacts.rake | 55 ------------------------------------------ rakelib/plugin.rake | 53 ++++++++++++++++++++++++++++++++++++++++ rubyUtils.gradle | 12 +++++++-- 3 files changed, 63 insertions(+), 57 deletions(-) diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 1856b39e6ae..27fd91d44bb 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -82,58 +82,6 @@ namespace "artifact" do res end - def duplicated_gems_exclude_paths - shared_gems_path = 'vendor/jruby/lib/ruby/gems/shared/gems' - default_gemspecs_path = 'vendor/jruby/lib/ruby/gems/shared/specifications/default' - bundle_gems_path = 'vendor/bundle/jruby/*/gems' - - exclusions = [] - - # "bundled" gems in jruby - # https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L108-L152 - shared_gem_names = Dir.glob(File.join(shared_gems_path, '*')).map do |path| - match = File.basename(path).match(/^(.+?)-\d+/) - match ? match[1] : nil - end.compact - - # "default" gems in jruby/ruby - # https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L21-L106 - default_gem_names = Dir.glob(File.join(default_gemspecs_path, '*.gemspec')).map do |path| - match = File.basename(path).match(/^(.+?)-\d+/) - match ? match[1] : nil - end.compact - - # gems we explicitly manage with bundler (we always want these to take precedence) - bundle_gem_names = Dir.glob(File.join(bundle_gems_path, '*')).map do |path| - match = File.basename(path).match(/^(.+?)-\d+/) - match ? match[1] : nil - end.compact - - shared_duplicates = shared_gem_names & bundle_gem_names - default_duplicates = default_gem_names & bundle_gem_names - all_duplicates = (shared_duplicates + default_duplicates).uniq - puts "Adding duplicate gems to exclude path: #{all_duplicates.sort.join(', ')}" - - # Exclude shared/bundled gems duplicates - shared_duplicates.each do |gem_name| - exclusions << "vendor/jruby/lib/ruby/gems/shared/gems/#{gem_name}-*/**/*" - exclusions << "vendor/jruby/lib/ruby/gems/shared/gems/#{gem_name}-*" - exclusions << "vendor/jruby/lib/ruby/gems/shared/specifications/#{gem_name}-*.gemspec" - end - - # Exclude default gems duplicates - default_duplicates.each do |gem_name| - # CODEREVIEW: removing the code itself causes issues with gem loading. Remove only the gemspecs - # for duplicated gems. This should help the code scanning case. - exclusions << "vendor/jruby/lib/ruby/gems/shared/specifications/default/#{gem_name}-*.gemspec" - # exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}.rb" - # exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}/**/*" - # exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}" - end - - exclusions - end - def default_exclude_paths return @exclude_paths if @exclude_paths @@ -153,9 +101,6 @@ namespace "artifact" do @exclude_paths << 'vendor/**/gems/**/Gemfile.lock' @exclude_paths << 'vendor/**/gems/**/Gemfile' - @exclude_paths.concat(duplicated_gems_exclude_paths) - puts "Full exclude_paths list:" - @exclude_paths.each { |path| puts " - #{path}" } @exclude_paths.freeze end diff --git a/rakelib/plugin.rake b/rakelib/plugin.rake index 47d572f417a..3742b59b8d5 100644 --- a/rakelib/plugin.rake +++ b/rakelib/plugin.rake @@ -90,12 +90,65 @@ namespace "plugin" do task.reenable # Allow this task to be run again end # task "install" + + task "clean-duplicate-gems" do + shared_gems_path = File.join(LogStash::Environment::LOGSTASH_HOME, + 'vendor/jruby/lib/ruby/gems/shared/gems') + default_gemspecs_path = File.join(LogStash::Environment::LOGSTASH_HOME, + 'vendor/jruby/lib/ruby/gems/shared/specifications/default') + bundle_gems_path = File.join(LogStash::Environment::BUNDLE_DIR, + 'jruby/*/gems') + + # "bundled" gems in jruby + # https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L108-L152 + shared_gem_names = Dir.glob(File.join(shared_gems_path, '*')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + # "default" gems in jruby/ruby + # https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L21-L106 + default_gem_names = Dir.glob(File.join(default_gemspecs_path, '*.gemspec')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + # gems we explicitly manage with bundler (we always want these to take precedence) + bundle_gem_names = Dir.glob(File.join(bundle_gems_path, '*')).map do |path| + match = File.basename(path).match(/^(.+?)-\d+/) + match ? match[1] : nil + end.compact + + shared_duplicates = shared_gem_names & bundle_gem_names + default_duplicates = default_gem_names & bundle_gem_names + all_duplicates = (shared_duplicates + default_duplicates).uniq + + puts("[plugin:clean-duplicate-gems] Removing duplicate gems: #{all_duplicates.sort.join(', ')}") + + # Remove shared/bundled gem duplicates + shared_duplicates.each do |gem_name| + FileUtils.rm_rf(Dir.glob("#{shared_gems_path}/#{gem_name}-*")) + FileUtils.rm_rf(Dir.glob("#{shared_gems_path}/../specifications/#{gem_name}-*.gemspec")) + end + + # Remove default gem gemspecs only + default_duplicates.each do |gem_name| + # For stdlib default gems we only remove the gemspecs as removing the source code + # files results in code loading errors and ruby warnings + FileUtils.rm_rf(Dir.glob("#{default_gemspecs_path}/#{gem_name}-*.gemspec")) + end + + task.reenable + end + task "install-default" => "bootstrap" do puts("[plugin:install-default] Installing default plugins") remove_lockfile # because we want to use the release lockfile install_plugins("--no-verify", "--preserve", *LogStash::RakeLib::DEFAULT_PLUGINS) + # Clean duplicates after full gem resolution + Rake::Task["plugin:clean-duplicate-gems"].invoke task.reenable # Allow this task to be run again end diff --git a/rubyUtils.gradle b/rubyUtils.gradle index 4e2565f72a4..99b5387b2d3 100644 --- a/rubyUtils.gradle +++ b/rubyUtils.gradle @@ -279,7 +279,11 @@ tasks.register("installCustomJRuby", Copy) { dependsOn buildCustomJRuby description = "Install custom built JRuby in the vendor directory" inputs.file(customJRubyTar) - outputs.dir("${projectDir}/vendor/jruby") + // Don't re-extract if core JRuby is already installed. This works around + // gem deduplication when rake calls back in to gradle. + onlyIf { + !file("${projectDir}/vendor/jruby/bin/jruby").exists() + } from tarTree(customJRubyTar == "" ? jrubyTarPath : customJRubyTar) eachFile { f -> f.path = f.path.replaceFirst("^jruby-${customJRubyVersion}", '') @@ -294,7 +298,11 @@ tasks.register("downloadAndInstallJRuby", Copy) { dependsOn=[verifyFile, installCustomJRuby] description = "Install JRuby in the vendor directory" inputs.file(jrubyTarPath) - outputs.dir("${projectDir}/vendor/jruby") + // Don't re-extract if core JRuby is already installed. This works around + // gem deduplication when rake calls back in to gradle. + onlyIf { + !file("${projectDir}/vendor/jruby/bin/jruby").exists() + } from tarTree(downloadJRuby.dest) eachFile { f -> f.path = f.path.replaceFirst("^jruby-${jRubyVersion}", '') From d6b0338993dbd91e1523fcf8563100c127a6c449 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Mon, 3 Nov 2025 16:04:35 -0800 Subject: [PATCH 04/11] Ensure the set of gems tested at unit level matches packages This commit adds the installDefaultGems task to the unit test tasks. This ensures that the gem env tested at the unit level matches the deduplicated one at the integration/acceptance level. Takes over https://github.com/elastic/logstash/pull/18330 --- build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle b/build.gradle index a238e899729..a89552fa617 100644 --- a/build.gradle +++ b/build.gradle @@ -418,6 +418,7 @@ project(":logstash-core") { tasks.getByPath(":logstash-core:" + tsk).configure { dependsOn copyPluginTestAlias dependsOn installDevelopmentGems + dependsOn installDefaultGems } } } @@ -1001,6 +1002,7 @@ if (System.getenv('OSS') != 'true') { ["rubyTests", "rubyIntegrationTests", "test"].each { tsk -> tasks.getByPath(":logstash-xpack:" + tsk).configure { dependsOn installDevelopmentGems + dependsOn installDefaultGems } } } From 5e6776d057819a36e2656a5675b09ba961f550d1 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Tue, 4 Nov 2025 16:24:58 -0800 Subject: [PATCH 05/11] WIP: Use logstash_gem_home for gemInstaller This commit changes gemInstaller such that the centralized gem_home from Logstash::Environment is used instead of hard coding in a fragile path. The tests were the only consumer of the optional positional parameter in the `install` class method. --- lib/pluginmanager/gem_installer.rb | 10 ++++---- .../unit/plugin_manager/gem_installer_spec.rb | 25 +++++++++++++------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/pluginmanager/gem_installer.rb b/lib/pluginmanager/gem_installer.rb index e5560a78025..17eb889ef8e 100644 --- a/lib/pluginmanager/gem_installer.rb +++ b/lib/pluginmanager/gem_installer.rb @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +require "bootstrap/environment" require "pluginmanager/ui" require "pathname" require "rubygems/package" @@ -26,17 +27,16 @@ module LogStash module PluginManager # - Generate the specifications # - Copy the data in the right folders class GemInstaller - GEM_HOME = Pathname.new(::File.join(LogStash::Environment::BUNDLE_DIR, "jruby", "3.1.0")) SPECIFICATIONS_DIR = "specifications" GEMS_DIR = "gems" CACHE_DIR = "cache" attr_reader :gem_home - def initialize(gem_file, display_post_install_message = false, gem_home = GEM_HOME) + def initialize(gem_file, display_post_install_message = false) @gem_file = gem_file @gem = ::Gem::Package.new(@gem_file) - @gem_home = Pathname.new(gem_home) + @gem_home = Pathname.new(LogStash::Environment.logstash_gem_home) @display_post_install_message = display_post_install_message end @@ -48,8 +48,8 @@ def install post_install_message end - def self.install(gem_file, display_post_install_message = false, gem_home = GEM_HOME) - self.new(gem_file, display_post_install_message, gem_home).install + def self.install(gem_file, display_post_install_message = false) + self.new(gem_file, display_post_install_message).install end private diff --git a/spec/unit/plugin_manager/gem_installer_spec.rb b/spec/unit/plugin_manager/gem_installer_spec.rb index dd85009dfff..89ee9c93a51 100644 --- a/spec/unit/plugin_manager/gem_installer_spec.rb +++ b/spec/unit/plugin_manager/gem_installer_spec.rb @@ -27,18 +27,27 @@ let(:simple_gem) { ::File.join(::File.dirname(__FILE__), "..", "..", "support", "pack", "valid-pack", "logstash", "valid-pack", "#{plugin_name}.gem") } subject { described_class } - let(:temporary_gem_home) { p = Stud::Temporary.pathname; FileUtils.mkdir_p(p); p } + let(:gem_home) { LogStash::Environment.logstash_gem_home } + # Clean up installed gems after each test + after(:each) do + spec_file = ::File.join(gem_home, "specifications", "#{plugin_name}.gemspec") + FileUtils.rm_f(spec_file) if ::File.exist?(spec_file) + gem_dir = ::File.join(gem_home, "gems", plugin_name) + FileUtils.rm_rf(gem_dir) if Dir.exist?(gem_dir) + cache_file = ::File.join(gem_home, "cache", "#{plugin_name}.gem") + FileUtils.rm_f(cache_file) if ::File.exist?(cache_file) + end it "install the specifications in the spec dir" do - subject.install(simple_gem, false, temporary_gem_home) - spec_file = ::File.join(temporary_gem_home, "specifications", "#{plugin_name}.gemspec") + subject.install(simple_gem, false) + spec_file = ::File.join(gem_home, "specifications", "#{plugin_name}.gemspec") expect(::File.exist?(spec_file)).to be_truthy expect(::File.size(spec_file)).to be > 0 end it "install the gem in the gems dir" do - subject.install(simple_gem, false, temporary_gem_home) - gem_dir = ::File.join(temporary_gem_home, "gems", plugin_name) + subject.install(simple_gem, false) + gem_dir = ::File.join(gem_home, "gems", plugin_name) expect(Dir.exist?(gem_dir)).to be_truthy end @@ -50,13 +59,13 @@ context "when we want the message" do it "display the message" do - expect(subject.install(simple_gem, true, temporary_gem_home)).to eq(message) + expect(subject.install(simple_gem, true)).to eq(message) end end context "when we dont want the message" do it "doesn't display the message" do - expect(subject.install(simple_gem, false, temporary_gem_home)).to be_nil + expect(subject.install(simple_gem, false)).to be_nil end end end @@ -65,7 +74,7 @@ context "when we don't want the message" do it "doesn't display the message" do expect(LogStash::PluginManager.ui).not_to receive(:info).with(message) - subject.install(simple_gem, true, temporary_gem_home) + subject.install(simple_gem, true) end end end From 7dc5da8907cc8845a9e80f0b6dececc198f29b88 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 5 Nov 2025 16:25:00 -0800 Subject: [PATCH 06/11] Fix gem env setup for ruby unit tests After some deeeeeeeep diving into comparing the state of running logstash from a compiled artifact vs the unit tests i finally figured out that the use of the bundler `setup!` method in unit tests is imcompatible with a couple of tests. Specifically that method puts bundler installed gems ahead of the standard lib gems in the load path. This commit solves that by re-positioning the standarl lib back to the front of the load path. --- lib/bootstrap/rspec.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/bootstrap/rspec.rb b/lib/bootstrap/rspec.rb index 782455135ee..db6cdeb7571 100755 --- a/lib/bootstrap/rspec.rb +++ b/lib/bootstrap/rspec.rb @@ -17,6 +17,16 @@ require_relative "environment" LogStash::Bundler.setup!({:without => [:build]}) +# Our use of LogStash::Bundler.setup! here leaves us in kind of a wonky state for *all* tests +# Essentially we end up with a load path that favors bundlers gem env over stdlib. This is +# not really the call stack in logstash itself, so while this does make the full bundled gem +# env available for tests, it also has a quirk where stdlib gems are not loaed correctly. The +# following patch ensures that stdlib gems are bumped to the front of the load path for unit +# tests. +## START PATCH ## +jruby_stdlib = $LOAD_PATH.find { |p| p.end_with?('vendor/jruby/lib/ruby/stdlib') } +$LOAD_PATH.unshift($LOAD_PATH.delete(jruby_stdlib)) if jruby_stdlib +## END PATCH ## require "logstash-core" require "logstash/environment" From d02fb9f21f2575543dc047d498c0c425790865d0 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 26 Nov 2025 12:25:00 -0800 Subject: [PATCH 07/11] Show how using `--prefer-local` causes issues Ideally bundler will consider default/stdlib gems when doing dependency resolution to avoid duplication in the first place. this seems to break the pluginmanager. Verify this happens in CI... --- lib/bootstrap/bundler.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bootstrap/bundler.rb b/lib/bootstrap/bundler.rb index f1c93cff45c..f3b57eb0ac4 100644 --- a/lib/bootstrap/bundler.rb +++ b/lib/bootstrap/bundler.rb @@ -280,6 +280,7 @@ def bundler_arguments(options = {}) arguments = [] if options[:install] arguments << "install" + arguments << "--prefer-local" arguments << "--clean" if options[:clean] if options[:local] arguments << "--local" From 4493ec383baf231b5ffa4ef3b1ac5dfe6eea008c Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 28 Nov 2025 14:28:39 +0000 Subject: [PATCH 08/11] Make prefer-local work well lib/bootstrap/bundler.rb: 1. Added jruby_bundled_specs method to Source::Rubygems that returns specs from JRuby's specifications/ directory 2. Modified specs method to merge jruby_bundled_specs with higher precedence when prefer_local is true 3. Updated install patch to skip installation for gems that already exist in JRuby's bundled gems directory 4. Track @jruby_bundled_specs_dir alongside @jruby_default_specs_dir lib/pluginmanager/command.rb: 1. Added explicit filter to not touch JRuby's bundled gems in remove_orphan_dependencies! The behavior now: - JRuby bundled gems are preferred over remote versions - If a version constraint requires a newer version, it will be fetched from remote - No unnecessary copying of already-available gems to vendor/bundle --- lib/bootstrap/bundler.rb | 288 +++++++++++++++++- .../bundler/logstash_uninstall.rb | 1 + lib/pluginmanager/command.rb | 5 + rakelib/plugin.rake | 6 +- rubyUtils.gradle | 9 +- 5 files changed, 296 insertions(+), 13 deletions(-) diff --git a/lib/bootstrap/bundler.rb b/lib/bootstrap/bundler.rb index f3b57eb0ac4..c56e7620c91 100644 --- a/lib/bootstrap/bundler.rb +++ b/lib/bootstrap/bundler.rb @@ -20,6 +20,10 @@ module Bundler extend self def patch! + + return if @bundler_patched + @bundler_patched = true + # Patch to prevent Bundler to save a .bundle/config file in the root # of the application ::Bundler::Settings.module_exec do @@ -40,15 +44,15 @@ def self.reset_paths! end end - # When preparing offline packs or generally when installing gems, bundler wants to have `.gem` files + # When preparing offline packs or generally when installing gems, bundler wants to have `.gem` files # cached. We ship a default set of gems that inclue all of the unpacked code. During dependency - # resolution bundler still wants to ensure`.gem` files exist. This patch updates two paths in bundler where + # resolution bundler still wants to ensure`.gem` files exist. This patch updates two paths in bundler where # it natively it would *fail* when a `.gem` file is not found. Instead of failing we force the cache to be # updated with a `.gem` file. This preserves the original patch behavior. There is still an open question of - # *how* to potentially update the files we vendor or the way we set up bundler to avoid carrying this patch. + # *how* to potentially update the files we vendor or the way we set up bundler to avoid carrying this patch. # As of JRuby 9.4.13.0 rubygems (bundler) is at 3.6.3. There have been some releases and changes in bundler code # since then but it does not seem to have changed the way it handles gem files. Obviously carrying a patch like this - # carries a maintenance burden so prioritizing a packaging solution may be + # carries a maintenance burden so prioritizing a packaging solution may be ::Bundler::Source::Rubygems.module_exec do def fetch_gem_if_possible(spec, previous_spec = nil) path = if spec.remote @@ -75,18 +79,278 @@ def cache(spec, custom_path = nil) raise InstallError, e.message end end + + # + # BACKPORT: Fix `--prefer-local` flag (from rubygems/rubygems commits 607a3bf479, 209b93a, 23047a0) + # + # The original implementation of --prefer-local was too naive: + # 1. It didn't pass prefer_local to Package objects + # 2. It returned empty array when no local specs exist (instead of falling back to remote) + # 3. It didn't properly handle default gems + # + # These patches fix: + # - PR #7951: Fix `--prefer-local` flag (propagate to packages, add fallback logic) + # - PR #8412: Fix `--prefer-local` not respecting default gems + # - PR #8484: Fix `bundle install --prefer-local` sometimes installing very old versions + # + + # Patch Source base class to add prefer_local! method + ::Bundler::Source.class_eval do + def prefer_local! + # Base implementation - does nothing, subclasses override + end + end + + # Patch Source::Rubygems to track prefer_local state and handle default_specs properly + # Also add support for JRuby bundled gems (gems in vendor/jruby/.../specifications/) + ::Bundler::Source::Rubygems.class_eval do + # Add prefer_local! method + def prefer_local! + @prefer_local = true + end + + # Return specs from JRuby's bundled gem directory (specifications/, not specifications/default/) + # These are gems that ship with JRuby but aren't "default gems" in the Ruby sense + def jruby_bundled_specs + @jruby_bundled_specs ||= begin + idx = ::Bundler::Index.new + jruby_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_bundled_specs_dir) + jruby_default_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_default_specs_dir) + + if jruby_specs_dir && ::File.directory?(jruby_specs_dir) + # Get gemspecs from specifications/ but NOT from specifications/default/ + ::Dir[::File.join(jruby_specs_dir, "*.gemspec")].each do |path| + # Skip if this is actually in the default directory + next if jruby_default_specs_dir && path.start_with?(jruby_default_specs_dir) + + stub = ::Gem::StubSpecification.gemspec_stub(path, jruby_specs_dir, jruby_specs_dir) + # Create a Bundler::StubSpecification from the Gem::StubSpecification + bundler_spec = ::Bundler::StubSpecification.from_stub(stub) + # Set source to self (the Source::Rubygems instance) - required for materialization + bundler_spec.source = self + idx << bundler_spec + end + end + idx + end + end + + # Override specs method to handle prefer_local for default_specs AND jruby_bundled_specs + alias_method :original_specs, :specs + + def specs + @specs ||= begin + # remote_specs usually generates a way larger Index than the other + # sources, and large_idx.merge! small_idx is way faster than + # small_idx.merge! large_idx. + index = @allow_remote ? remote_specs.dup : ::Bundler::Index.new + index.merge!(cached_specs) if @allow_cached + index.merge!(installed_specs) if @allow_local + + if @allow_local + if @prefer_local + # With prefer_local, merge jruby_bundled_specs and default_specs so they take precedence + # over remote/cached/installed specs. This ensures JRuby's bundled gems are preferred. + index.merge!(jruby_bundled_specs) + index.merge!(default_specs) + else + # complete with default specs, only if not already available in the + # index through remote, cached, or installed specs + index.use(jruby_bundled_specs) + index.use(default_specs) + end + end + + index + end + end + end + + # Patch SourceList to propagate prefer_local! to all sources + ::Bundler::SourceList.class_eval do + def prefer_local! + all_sources.each(&:prefer_local!) + end + end + + # Patch Definition to call sources.prefer_local! when prefer_local! is called + ::Bundler::Definition.class_eval do + alias_method :original_prefer_local!, :prefer_local! + + def prefer_local! + @prefer_local = true + sources.prefer_local! + end + end + + # Patch Package to add prefer_local support + ::Bundler::Resolver::Package.class_eval do + def prefer_local? + @prefer_local + end + + def consider_remote_versions! + @prefer_local = false + end + end + + # Patch Resolver::Base to propagate prefer_local to packages and add include_remote_specs + ::Bundler::Resolver::Base.class_eval do + alias_method :original_base_initialize, :initialize + + def initialize(source_requirements, dependencies, base, platforms, options) + @prefer_local_option = options[:prefer_local] + original_base_initialize(source_requirements, dependencies, base, platforms, options) + end + + alias_method :original_get_package, :get_package + + def get_package(name) + package = original_get_package(name) + # Inject prefer_local into packages since older Bundler doesn't pass it through + if @prefer_local_option && !package.instance_variable_get(:@prefer_local) + package.instance_variable_set(:@prefer_local, true) + end + package + end + + def include_remote_specs(names) + names.each do |name| + get_package(name).consider_remote_versions! + end + end + end + + # Patch Resolver to fix filter_remote_specs with proper fallback + ::Bundler::Resolver.class_eval do + # Override filter_remote_specs with the fixed version from Bundler 2.7+ + # This fixes the issue where --prefer-local would return empty specs + # when no local gems are installed, instead of falling back to remote + def filter_remote_specs(specs, package) + if package.prefer_local? + local_specs = specs.select {|s| s.is_a?(::Bundler::StubSpecification) } + + if local_specs.empty? + # BACKPORT FIX: If no local specs exist, fall back to remote specs + # instead of returning empty array + package.consider_remote_versions! + specs + else + local_specs + end + else + specs + end + end + end + + # Patch Source::Rubygems#install to skip installation for default gems and JRuby bundled gems + # The original condition `spec.default_gem? && !cached_built_in_gem(...)` has a side effect: + # cached_built_in_gem fetches from remote if not in cache. For default gems and JRuby bundled gems, + # we should skip installation entirely without needing a cached .gem file. + ::Bundler::Source::Rubygems.class_eval do + alias_method :original_rubygems_install, :install + + def install(spec, options = {}) + # For default gems, skip installation entirely - they're already available + if spec.default_gem? + print_using_message "Using #{version_message(spec, options[:previous_spec])}" + return nil + end + + # For JRuby bundled gems, also skip installation - they're already available + # Check if this exact gem (name + version) exists in JRuby's bundled gems + jruby_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_bundled_specs_dir) + if jruby_specs_dir && ::File.directory?(jruby_specs_dir) + jruby_gemspec_path = ::File.join(jruby_specs_dir, "#{spec.name}-#{spec.version}.gemspec") + if ::File.exist?(jruby_gemspec_path) + print_using_message "Using #{version_message(spec, options[:previous_spec])}" + return nil + end + end + + original_rubygems_install(spec, options) + end + end end + # Capture JRuby's default gem directory before paths are changed + # This is needed so that default gems (like json) can be found with --prefer-local + def preserve_jruby_default_gems_path + return @jruby_default_gem_dir if defined?(@jruby_default_gem_dir) + + # The Gradle/JRuby setup already changes Gem.default_dir to a temp path before + # this code runs, so we need to construct the actual JRuby path from LOGSTASH_HOME + logstash_home = ENV["LOGSTASH_HOME"] || ::File.expand_path("../../..", __FILE__) + jruby_gems_dir = ::File.join(logstash_home, "vendor", "jruby", "lib", "ruby", "gems", "shared") + jruby_default_specs = ::File.join(jruby_gems_dir, "specifications", "default") + jruby_bundled_specs = ::File.join(jruby_gems_dir, "specifications") + + if ::File.directory?(jruby_default_specs) + @jruby_default_gem_dir = jruby_gems_dir + @jruby_default_specs_dir = jruby_default_specs + @jruby_bundled_specs_dir = jruby_bundled_specs + else + # Fall back to Gem.default_dir if vendor/jruby doesn't exist + @jruby_default_gem_dir = ::Gem.default_dir + @jruby_default_specs_dir = ::Gem.default_specifications_dir + @jruby_bundled_specs_dir = nil + end + + @jruby_default_gem_dir + end + + # Patch Gem::Specification.default_stubs to also look in JRuby's original default specs directory + # This is needed because Gem.default_specifications_dir only returns a single path, + # and after Gem.paths = ENV it points to Logstash's gem home, not JRuby's installation + def patch_default_stubs! + return if @default_stubs_patched || !defined?(@jruby_default_specs_dir) || @jruby_default_specs_dir.nil? + @default_stubs_patched = true + + jruby_specs_dir = @jruby_default_specs_dir + + ::Gem::Specification.singleton_class.class_eval do + alias_method :original_default_stubs, :default_stubs + + define_method(:default_stubs) do |pattern = "*.gemspec"| + # Get stubs from the current default_specifications_dir + stubs = original_default_stubs(pattern) + + # Also look in JRuby's original default specs directory if it exists and is different + if jruby_specs_dir && ::File.directory?(jruby_specs_dir) && jruby_specs_dir != ::Gem.default_specifications_dir + base_dir = jruby_specs_dir + ::Dir[::File.join(base_dir, pattern)].each do |path| + # Use default_gemspec_stub to mark these as default gems (default_gem = true) + stub = ::Gem::StubSpecification.default_gemspec_stub(path, base_dir, base_dir) + stubs << stub unless stubs.any? { |s| s.name == stub.name && s.version == stub.version } + end + end + + stubs + end + end + end + # prepare bundler's environment variables, but do not invoke ::Bundler::setup def prepare(options = {}) options = {:without => [:development]}.merge(options) options[:without] = Array(options[:without]) + # Capture JRuby default gems path BEFORE clearing + jruby_gem_dir = preserve_jruby_default_gems_path + ::Gem.clear_paths - ENV['GEM_HOME'] = ENV['GEM_PATH'] = Environment.logstash_gem_home + # Include both Logstash gem home AND JRuby's default gem directory in GEM_PATH + # This ensures default gems can be discovered by Gem::Specification.default_stubs + gem_path = [Environment.logstash_gem_home, jruby_gem_dir].compact.uniq.join(::File::PATH_SEPARATOR) + ENV['GEM_HOME'] = Environment.logstash_gem_home + ENV['GEM_PATH'] = gem_path ::Gem.paths = ENV + # Patch default_stubs to also look in JRuby's original location + patch_default_stubs! + # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s) ENV["BUNDLE_GEMFILE"] = Environment::GEMFILE_PATH @@ -127,9 +391,21 @@ def invoke!(options = {}) :jobs => 12, :all => false, :package => false, :without => [:development]}.merge(options) options[:without] = Array(options[:without]) options[:update] = Array(options[:update]) if options[:update] + + # Capture JRuby default gems path BEFORE clearing + jruby_gem_dir = preserve_jruby_default_gems_path + ::Gem.clear_paths - ENV['GEM_HOME'] = ENV['GEM_PATH'] = LogStash::Environment.logstash_gem_home + # Include both Logstash gem home AND JRuby's default gem directory in GEM_PATH + # This ensures default gems can be discovered by Gem::Specification.default_stubs + gem_path = [LogStash::Environment.logstash_gem_home, jruby_gem_dir].compact.uniq.join(::File::PATH_SEPARATOR) + ENV['GEM_HOME'] = LogStash::Environment.logstash_gem_home + ENV['GEM_PATH'] = gem_path ::Gem.paths = ENV + + # Patch default_stubs to also look in JRuby's original location + patch_default_stubs! + # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s). # in the context of calling Bundler::CLI this is not really required since Bundler::CLI will look at # Bundler.settings[:gemfile] unlike Bundler.setup. For the sake of consistency and defensive/future proofing, let's keep it here. diff --git a/lib/pluginmanager/bundler/logstash_uninstall.rb b/lib/pluginmanager/bundler/logstash_uninstall.rb index 996e840cf27..1897fed5b8b 100644 --- a/lib/pluginmanager/bundler/logstash_uninstall.rb +++ b/lib/pluginmanager/bundler/logstash_uninstall.rb @@ -35,6 +35,7 @@ def initialize(gemfile_path, lockfile_path) end def uninstall!(gems_to_remove) + gems_to_remove = Array(gems_to_remove) unsatisfied_dependency_mapping = Dsl.evaluate(gemfile_path, lockfile_path, {}).specs.each_with_object({}) do |spec, memo| diff --git a/lib/pluginmanager/command.rb b/lib/pluginmanager/command.rb index 5a7c1a51f97..70d7d680a47 100644 --- a/lib/pluginmanager/command.rb +++ b/lib/pluginmanager/command.rb @@ -50,9 +50,14 @@ def remove_unused_locally_installed_gems! def remove_orphan_dependencies! locked_gem_names = ::Bundler::LockfileParser.new(File.read(LogStash::Environment::LOCKFILE)).specs.map(&:full_name).to_set + bundle_path = LogStash::Environment::BUNDLE_DIR + # JRuby bundled gems path - never touch these + jruby_gems_path = File.join(LogStash::Environment::LOGSTASH_HOME, "vendor", "jruby", "lib", "ruby", "gems") orphan_gem_specs = ::Gem::Specification.each .reject(&:stubbed?) # skipped stubbed (uninstalled) gems .reject(&:default_gem?) # don't touch jruby-included default gems + .reject { |spec| spec.full_gem_path.start_with?(jruby_gems_path) } # don't touch jruby bundled gems + .select { |spec| spec.full_gem_path.start_with?(bundle_path) } # only gems in bundle path .reject{ |spec| locked_gem_names.include?(spec.full_name) } .sort diff --git a/rakelib/plugin.rake b/rakelib/plugin.rake index 3742b59b8d5..d37ca541d01 100644 --- a/rakelib/plugin.rake +++ b/rakelib/plugin.rake @@ -90,7 +90,6 @@ namespace "plugin" do task.reenable # Allow this task to be run again end # task "install" - task "clean-duplicate-gems" do shared_gems_path = File.join(LogStash::Environment::LOGSTASH_HOME, 'vendor/jruby/lib/ruby/gems/shared/gems') @@ -133,14 +132,15 @@ namespace "plugin" do # Remove default gem gemspecs only default_duplicates.each do |gem_name| - # For stdlib default gems we only remove the gemspecs as removing the source code + # For stdlib default gems we only remove the gemspecs as removing the source code # files results in code loading errors and ruby warnings FileUtils.rm_rf(Dir.glob("#{default_gemspecs_path}/#{gem_name}-*.gemspec")) end - + task.reenable end + task "install-default" => "bootstrap" do puts("[plugin:install-default] Installing default plugins") diff --git a/rubyUtils.gradle b/rubyUtils.gradle index 99b5387b2d3..0eb244ae96c 100644 --- a/rubyUtils.gradle +++ b/rubyUtils.gradle @@ -176,7 +176,7 @@ void setupJruby(File projectDir, File buildDir) { executeJruby projectDir, buildDir, { ScriptingContainer jruby -> jruby.currentDirectory = projectDir jruby.runScriptlet("require '${projectDir}/lib/bootstrap/environment'") - jruby.runScriptlet("LogStash::Bundler.invoke!") + jruby.runScriptlet("LogStash::Bundler.invoke!(:install => true)") jruby.runScriptlet("LogStash::Bundler.genericize_platform") } } @@ -190,12 +190,13 @@ void setupJruby(File projectDir, File buildDir) { Object executeJruby(File projectDir, File buildDir, Closure /* Object*/ block) { def jruby = new ScriptingContainer() def env = jruby.environment - def gemDir = "${projectDir}/vendor/bundle/jruby/3.1.0".toString() + def gemHomeDir = "${projectDir}/vendor/bundle/jruby/3.1.0".toString() + def gemPathDir = "${projectDir}/vendor/jruby/lib/ruby/gems/shared:${projectDir}/vendor/bundle/jruby/3.1.0".toString() jruby.setLoadPaths(["${projectDir}/vendor/jruby/lib/ruby/stdlib".toString()]) env.put "USE_RUBY", "1" - env.put "GEM_HOME", gemDir + env.put "GEM_HOME", gemHomeDir env.put "GEM_SPEC_CACHE", "${buildDir}/cache".toString() - env.put "GEM_PATH", gemDir + env.put "GEM_PATH", gemPathDir // Pass through ORG_GRADLE_PROJECT_fedrampHighMode if it exists in the project properties // See https://docs.gradle.org/current/userguide/build_environment.html#setting_a_project_property // For more information about setting properties via env vars prefixed with ORG_GRADLE_PROJECT From b3c846290169254464bc32b73ec56886c01bf029 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Dec 2025 13:39:58 +0000 Subject: [PATCH 09/11] minor fix --- lib/bootstrap/bundler.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/bootstrap/bundler.rb b/lib/bootstrap/bundler.rb index c56e7620c91..5a47f1af145 100644 --- a/lib/bootstrap/bundler.rb +++ b/lib/bootstrap/bundler.rb @@ -114,16 +114,23 @@ def prefer_local! def jruby_bundled_specs @jruby_bundled_specs ||= begin idx = ::Bundler::Index.new + jruby_gem_home = LogStash::Bundler.instance_variable_get(:@jruby_default_gem_dir) jruby_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_bundled_specs_dir) jruby_default_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_default_specs_dir) - if jruby_specs_dir && ::File.directory?(jruby_specs_dir) + if jruby_gem_home && jruby_specs_dir && ::File.directory?(jruby_specs_dir) + # gems_dir is where the actual gem code lives (gem_home/gems/) + jruby_gems_dir = ::File.join(jruby_gem_home, "gems") + # Get gemspecs from specifications/ but NOT from specifications/default/ ::Dir[::File.join(jruby_specs_dir, "*.gemspec")].each do |path| # Skip if this is actually in the default directory next if jruby_default_specs_dir && path.start_with?(jruby_default_specs_dir) - stub = ::Gem::StubSpecification.gemspec_stub(path, jruby_specs_dir, jruby_specs_dir) + # gemspec_stub params: filename, base_dir, gems_dir + # base_dir = gem home (parent of specifications/) + # gems_dir = where gem code lives (gem_home/gems/) + stub = ::Gem::StubSpecification.gemspec_stub(path, jruby_gem_home, jruby_gems_dir) # Create a Bundler::StubSpecification from the Gem::StubSpecification bundler_spec = ::Bundler::StubSpecification.from_stub(stub) # Set source to self (the Source::Rubygems instance) - required for materialization From 637682ca542b9344fc1411521a7fcfa28e490b07 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Dec 2025 14:38:31 +0000 Subject: [PATCH 10/11] minor test fix --- spec/unit/bootstrap/bundler_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/unit/bootstrap/bundler_spec.rb b/spec/unit/bootstrap/bundler_spec.rb index 44b65207d7a..b0d5f0d1b64 100644 --- a/spec/unit/bootstrap/bundler_spec.rb +++ b/spec/unit/bootstrap/bundler_spec.rb @@ -62,7 +62,9 @@ expect(::Bundler.settings[:gemfile]).to eq(LogStash::Environment::GEMFILE_PATH) expect(::Bundler.settings[:without]).to eq(options.fetch(:without, [])) - expect(ENV['GEM_PATH']).to eq(LogStash::Environment.logstash_gem_home) + # GEM_PATH includes both logstash gem home and JRuby's gem directory + # so that JRuby bundled gems (like rexml) can be found + expect(ENV['GEM_PATH']).to start_with(LogStash::Environment.logstash_gem_home) $stderr = original_stderr end From c2fb7946ef96e198bb3191f9d1c3b55d90e5ba9d Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Dec 2025 19:03:27 +0000 Subject: [PATCH 11/11] yet another fix --- lib/bootstrap/bundler.rb | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/bootstrap/bundler.rb b/lib/bootstrap/bundler.rb index 5a47f1af145..9403875c7b3 100644 --- a/lib/bootstrap/bundler.rb +++ b/lib/bootstrap/bundler.rb @@ -308,14 +308,19 @@ def preserve_jruby_default_gems_path @jruby_default_gem_dir end - # Patch Gem::Specification.default_stubs to also look in JRuby's original default specs directory + # Patch Gem::Specification.default_stubs to also look in JRuby's original specs directories # This is needed because Gem.default_specifications_dir only returns a single path, # and after Gem.paths = ENV it points to Logstash's gem home, not JRuby's installation + # We include BOTH: + # - specifications/default/ (true default gems) + # - specifications/ (JRuby bundled gems like rexml, rake, net-imap, etc.) def patch_default_stubs! return if @default_stubs_patched || !defined?(@jruby_default_specs_dir) || @jruby_default_specs_dir.nil? @default_stubs_patched = true - jruby_specs_dir = @jruby_default_specs_dir + jruby_default_specs_dir = @jruby_default_specs_dir + jruby_bundled_specs_dir = @jruby_bundled_specs_dir + jruby_gem_home = @jruby_default_gem_dir ::Gem::Specification.singleton_class.class_eval do alias_method :original_default_stubs, :default_stubs @@ -324,12 +329,26 @@ def patch_default_stubs! # Get stubs from the current default_specifications_dir stubs = original_default_stubs(pattern) - # Also look in JRuby's original default specs directory if it exists and is different - if jruby_specs_dir && ::File.directory?(jruby_specs_dir) && jruby_specs_dir != ::Gem.default_specifications_dir - base_dir = jruby_specs_dir - ::Dir[::File.join(base_dir, pattern)].each do |path| + # Also look in JRuby's original default specs directory (specifications/default/) + if jruby_default_specs_dir && ::File.directory?(jruby_default_specs_dir) && jruby_default_specs_dir != ::Gem.default_specifications_dir + ::Dir[::File.join(jruby_default_specs_dir, pattern)].each do |path| # Use default_gemspec_stub to mark these as default gems (default_gem = true) - stub = ::Gem::StubSpecification.default_gemspec_stub(path, base_dir, base_dir) + stub = ::Gem::StubSpecification.default_gemspec_stub(path, jruby_default_specs_dir, jruby_default_specs_dir) + stubs << stub unless stubs.any? { |s| s.name == stub.name && s.version == stub.version } + end + end + + # Also include JRuby's bundled gems (specifications/, excluding specifications/default/) + # These are gems like rexml, rake, net-imap that ship with JRuby but aren't "default gems" + # We treat them as default gems here so they're available via Bundler's add_default_gems_to + if jruby_bundled_specs_dir && jruby_gem_home && ::File.directory?(jruby_bundled_specs_dir) + jruby_gems_dir = ::File.join(jruby_gem_home, "gems") + ::Dir[::File.join(jruby_bundled_specs_dir, pattern)].each do |path| + # Skip if this is in the default directory (already handled above) + next if jruby_default_specs_dir && path.start_with?(jruby_default_specs_dir) + + # Use gemspec_stub (not default_gemspec_stub) with correct base_dir and gems_dir + stub = ::Gem::StubSpecification.gemspec_stub(path, jruby_gem_home, jruby_gems_dir) stubs << stub unless stubs.any? { |s| s.name == stub.name && s.version == stub.version } end end