Merge pull request #39 from salsify/ms/pimcore-3160

* Move on cancellation logic to after failure hook.
* Make delayed job group plugin incompatible with DelayedJob option `destroy_failed_jobs`.
* Wrap job group cancellation with lock to prevent multiple on cancellation jobs from being enqueued.

Author: marcoserrato

Appraisals

@@ -4,6 +4,7 @@ appraise 'rails-7.0' do
   gem 'activerecord', '~> 7.0.8'
   gem 'activesupport', '~> 7.0.8'
   gem 'sqlite3', '~> 1.7'
+  gem 'concurrent-ruby', '1.3.4'
 end
 
 appraise 'rails-7.1' do

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.13.0
+- Moves `on_cancellation` logic from the before delayed job lifecycle hook to the after hook.
+- Gem will now fail to load if `Delayed::Worker.destroy_failed_jobs` is set to true.
+- Wrapped the job group cancel hook in a lock to prevent concurrently failing jobs from enqueueing
+  multiple on cancellation jobs.
+
 ## 0.12.0
 - Add support for Rails 8.0.
 - Drop support for Rails 6.1

gemfiles/rails_7.0.gemfile

@@ -5,5 +5,6 @@ source "https://rubygems.org"
 gem "activerecord", "~> 7.0.8"
 gem "activesupport", "~> 7.0.8"
 gem "sqlite3", "~> 1.7"
+gem 'concurrent-ruby', '1.3.4'
 
 gemspec path: "../"

lib/delayed/job_groups/errors.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Delayed
+  module JobGroups
+    ConfigurationError = Class.new(StandardError)
+
+    class IncompatibleWithDelayedJobError < ConfigurationError
+      DEFAULT_MESSAGE = 'DelayedJobGroupsPlugin is incompatible with Delayed::Job ' \
+                        'when `destroy_failed_jobs` is set to `true`'
+
+      def initialize(msg = DEFAULT_MESSAGE)
+        super(msg)
+      end
+    end
+  end
+end

lib/delayed/job_groups/job_group.rb

@@ -62,8 +62,18 @@ def unblock
       end
 
       def cancel
-        Delayed::Job.enqueue(on_cancellation_job, on_cancellation_job_options || {}) if on_cancellation_job
-        destroy
+        with_lock do
+          return if destroyed?
+
+          Delayed::Job.enqueue(on_cancellation_job, on_cancellation_job_options || {}) if on_cancellation_job
+          destroy
+        end
+      rescue ActiveRecord::RecordNotFound
+        # The first failing job to attempt cancelling the job group will enqueue the
+        # on cancellation job and destroy the group. Any other concurrently failing job
+        # in the same group can therefore silently return if the job group has already
+        # been destroyed.
+        nil
       end
 
       def check_for_completion(skip_pending_jobs_check: false)

lib/delayed/job_groups/plugin.rb

@@ -17,7 +17,13 @@ def job.max_attempts
           end
         end
 
-        lifecycle.before(:failure) do |_worker, job|
+        # In order to allow individual jobs in the JobGroup to perform cleanup activities upon
+        # job failure, a JobGroups on cancellation job must be enqueued in the after failure
+        # delayed job lifecycle hook. If both were to be performed in the same hook, the job's
+        # failure hook and the on cancellation job may run at the same time since hook execution
+        # order is never guaranteed. This is particular important if the on cancellation job
+        # depends on state set in the failure hook of an individual job.
+        lifecycle.after(:failure) do |_worker, job|
           # If a job in the job group fails, then cancel the whole job group.
           # Need to check that the job group is present since another
           # job may have concurrently cancelled it.

lib/delayed/job_groups/railtie.rb

@@ -4,6 +4,15 @@ module Delayed
   module JobGroups
     class Railtie < ::Rails::Railtie
       config.after_initialize do
+
+        # On cancellation checks are performed in the after failure delayed job lifecycle, however
+        # https://github.com/collectiveidea/delayed_job/blob/master/lib/delayed/worker.rb#L268 may
+        # delete jobs before the hook runs. This could cause a successful job in the same group to
+        # complete the group instead of the group being cancelled. Therefore, we must ensure that
+        # the Delayed::Worker.destroy_failed_jobs is set to false, guaranteeing that the group is
+        # never empty if failure occurs.
+        raise Delayed::JobGroups::IncompatibleWithDelayedJobError if Delayed::Worker.destroy_failed_jobs
+
         Delayed::Backend::ActiveRecord::Job.include(Delayed::JobGroups::JobExtensions)
       end
     end

lib/delayed/job_groups/version.rb

@@ -2,6 +2,6 @@
 
 module Delayed
   module JobGroups
-    VERSION = '0.12.0'
+    VERSION = '0.13.0'
   end
 end

lib/delayed_job_groups_plugin.rb

@@ -4,6 +4,7 @@
 require 'active_record'
 require 'delayed_job'
 require 'delayed_job_active_record'
+require 'delayed/job_groups/errors'
 require 'delayed/job_groups/compatibility'
 require 'delayed/job_groups/complete_stuck_job_groups_job'
 require 'delayed/job_groups/job_extensions'
@@ -18,6 +19,7 @@
 else
   # Do the same as in the railtie
   Delayed::Backend::ActiveRecord::Job.include(Delayed::JobGroups::JobExtensions)
+  raise Delayed::JobGroups::IncompatibleWithDelayedJobError if Delayed::Worker.destroy_failed_jobs
 end
 
 Delayed::Worker.plugins << Delayed::JobGroups::Plugin

spec/delayed/job_groups/job_group_spec.rb

@@ -299,11 +299,28 @@
   end
 
   describe "#cancel" do
+    subject(:job_group) do
+      create(
+        :job_group,
+        on_completion_job: on_completion_job,
+        on_completion_job_options: on_completion_job_options,
+        on_cancellation_job: on_cancellation_job,
+        on_cancellation_job_options: on_cancellation_job_options,
+        blocked: blocked
+      )
+    end
+
     let!(:queued_job) { Delayed::Job.create!(job_group_id: job_group.id) }
     let!(:running_job)  { Delayed::Job.create!(job_group_id: job_group.id, locked_at: Time.now, locked_by: 'test') }
+    let(:before_hook) {}
+    let(:on_cancellation_job) { 'dummy job' }
+    let(:on_cancellation_job_options) do
+      { foo: 'bar' }
+    end
+    let(:cancel) { true }
 
     before do
-      job_group.cancel
+      job_group.cancel if cancel
     end
 
     it "destroys the job group" do
@@ -317,6 +334,21 @@
     it "does not destroy running jobs" do
       expect(running_job).not_to have_been_destroyed
     end
+
+    context "when already cancelled" do
+      let(:cancel) { false }
+
+      it "skips cancel block if already cancelled" do
+        other = Delayed::JobGroups::JobGroup.find(job_group.id)
+        job_group.cancel
+
+        other.cancel
+        other.cancel
+
+        expect(Delayed::Job)
+          .to have_received(:enqueue).with(on_cancellation_job, on_cancellation_job_options).once
+      end
+    end
   end
 
   describe "#failure_cancels_group?" do