91367 Debt Resolution Exhausted Metrics (#18472)

* 91367 Debt Resolution Exhausted Metrics * Add back old metrics for backwards compatability * Add notes * Remove file * Remove file * Add updates * Add updates * Add lib update * Add updates * Remove uneeded block * Add modules * Add updates * Add updates * Add udpates * Remove specs * Add updates * Add linter fixes * Reorganize changes to not exceed PR size limit Revert "Remove file" This reverts commit b9a68af. Revert "Remove file" This reverts commit 3e2c740. Revert "Remove specs" This reverts commit 3e18277. * Fix linter * Fix unit test * Feedback * Missed comment --------- Co-authored-by: Jun Tao Luo <[email protected]>
department-of-veterans-affairs · Oct 8, 2024 · e4b703c · e4b703c
1 parent a03a4df
commit e4b703c
Show file tree

Hide file tree

Showing 11 changed files with 176 additions and 12 deletions.
diff --git a/app/sidekiq/copay_notifications/new_statement_notification_job.rb b/app/sidekiq/copay_notifications/new_statement_notification_job.rb
@@ -29,7 +29,8 @@ class NewStatementNotificationJob
     end
 
     sidekiq_retries_exhausted do |_msg, ex|
-      StatsD.increment("#{STATSD_KEY_PREFIX}.failure")
+      StatsD.increment("#{STATSD_KEY_PREFIX}.failure") # remove when we get more data into the retries_exhausted below
+      StatsD.increment("#{STATSD_KEY_PREFIX}.retries_exhausted")
       Rails.logger.error <<~LOG
         NewStatementNotificationJob retries exhausted:
         Exception: #{ex.class} - #{ex.message}

diff --git a/app/sidekiq/copay_notifications/parse_new_statements_job.rb b/app/sidekiq/copay_notifications/parse_new_statements_job.rb
@@ -10,9 +10,19 @@ class ParseNewStatementsJob
     JOB_INTERVAL = Settings.mcp.notifications.job_interval
     # number of jobs to perform at next interval
     BATCH_SIZE = Settings.mcp.notifications.batch_size
+    STATSD_KEY_PREFIX = 'api.copay_notifications.json_file'
+
+    sidekiq_retries_exhausted do |_msg, ex|
+      StatsD.increment("#{STATSD_KEY_PREFIX}.retries_exhausted")
+      Rails.logger.error <<~LOG
+        CopayNotifications::ParseNewStatementsJob retries exhausted:
+        Exception: #{ex.class} - #{ex.message}
+        Backtrace: #{ex.backtrace.join("\n")}
+      LOG
+    end
 
     def perform(statements_json_byte)
-      StatsD.increment('api.copay_notifications.json_file.total')
+      StatsD.increment("#{STATSD_KEY_PREFIX}.total")
       # Decode and parse large json file (~60-90k objects)
       statements_json = Oj.load(Base64.decode64(statements_json_byte))
       unique_statements = statements_json.uniq { |statement| statement['veteranIdentifier'] }

diff --git a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb
@@ -12,8 +12,16 @@ class V0::Form5655::VBASubmissionJob
 
     class MissingUserAttributesError < StandardError; end
 
-    sidekiq_retries_exhausted do |job, _ex|
+    sidekiq_retries_exhausted do |job, ex|
+      StatsD.increment("#{STATS_KEY}.retries_exhausted")
+      submission_id = job['args'][0]
       user_uuid = job['args'][1]
+      Rails.logger.error <<~LOG
+        V0::Form5655::VBASubmissionJob retries exhausted:
+        submission_id: #{submission_id} | user_id: #{user_uuid}
+        Exception: #{ex.class} - #{ex.message}
+        Backtrace: #{ex.backtrace.join("\n")}
+      LOG
       UserProfileAttributes.find(user_uuid)&.destroy
     end
 

diff --git a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb
@@ -13,7 +13,15 @@ class V0::Form5655::VHASubmissionJob
     class MissingUserAttributesError < StandardError; end
 
     sidekiq_retries_exhausted do |job, _ex|
+      StatsD.increment("#{STATS_KEY}.retries_exhausted")
+      submission_id = job['args'][0]
       user_uuid = job['args'][1]
+      Rails.logger.error <<~LOG
+        V0::Form5655::VHASubmissionJob retries exhausted:
+        Exception: #{ex.class} - #{ex.message}
+        Backtrace: #{ex.backtrace.join("\n")}
+        submission_id: #{submission_id} | user_id: #{user_uuid}
+      LOG
       UserProfileAttributes.find(user_uuid)&.destroy
     end
 

diff --git a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb
@@ -11,7 +11,8 @@ class V0::Form5655::VHA::SharepointSubmissionJob
     sidekiq_options retry: 4
 
     sidekiq_retries_exhausted do |job, _ex|
-      StatsD.increment("#{STATS_KEY}.failure")
+      StatsD.increment("#{STATS_KEY}.failure") # Deprecate this in favor of exhausted naming convention below
+      StatsD.increment("#{STATS_KEY}.retries_exhausted")
       submission_id = job['args'][0]
       submission = DebtsApi::V0::Form5655Submission.find(submission_id)
       submission.register_failure("SharePoint Submission Failed: #{job['error_message']}.")

diff --git a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb
@@ -13,7 +13,8 @@ class V0::Form5655::VHA::VBSSubmissionJob
     class MissingUserAttributesError < StandardError; end
 
     sidekiq_retries_exhausted do |job, _ex|
-      StatsD.increment("#{STATS_KEY}.failure")
+      StatsD.increment("#{STATS_KEY}.failure") # Deprecate this in favor of exhausted naming convention below
+      StatsD.increment("#{STATS_KEY}.retries_exhausted")
       submission_id = job['args'][0]
       user_uuid = job['args'][1]
       UserProfileAttributes.find(user_uuid)&.destroy

diff --git a/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb b/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb
@@ -38,5 +38,61 @@
         expect(form_submission.error_message).to eq('uhoh')
       end
     end
+
+    context 'with retries exhausted' do
+      let(:config) { described_class }
+      let(:missing_attributes_exception) do
+        e = DebtsApi::V0::Form5655::VBASubmissionJob::MissingUserAttributesError.new('abc-123')
+        allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2])
+        e
+      end
+
+      let(:standard_exception) do
+        e = StandardError.new('abc-123')
+        allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2])
+        e
+      end
+
+      let(:msg) do
+        {
+          'class' => 'YourJobClassName',
+          'args' => %w[123 123-abc],
+          'jid' => '12345abcde',
+          'retry_count' => 5
+        }
+      end
+
+      it 'handles MissingUserAttributesError' do
+        expected_log_message = <<~LOG
+          V0::Form5655::VBASubmissionJob retries exhausted:
+          submission_id: 123 | user_id: 123-abc
+          Exception: #{missing_attributes_exception.class} - #{missing_attributes_exception.message}
+          Backtrace: #{missing_attributes_exception.backtrace.join("\n")}
+        LOG
+
+        expect(StatsD).to receive(:increment).with(
+          "#{DebtsApi::V0::Form5655::VBASubmissionJob::STATS_KEY}.retries_exhausted"
+        )
+
+        expect(Rails.logger).to receive(:error).with(expected_log_message)
+        config.sidekiq_retries_exhausted_block.call(msg, missing_attributes_exception)
+      end
+
+      it 'handles unexpected errors' do
+        expected_log_message = <<~LOG
+          V0::Form5655::VBASubmissionJob retries exhausted:
+          submission_id: 123 | user_id: 123-abc
+          Exception: #{standard_exception.class} - #{standard_exception.message}
+          Backtrace: #{standard_exception.backtrace.join("\n")}
+        LOG
+
+        expect(StatsD).to receive(:increment).with(
+          "#{DebtsApi::V0::Form5655::VBASubmissionJob::STATS_KEY}.retries_exhausted"
+        )
+
+        expect(Rails.logger).to receive(:error).with(expected_log_message)
+        config.sidekiq_retries_exhausted_block.call(msg, standard_exception)
+      end
+    end
   end
 end
diff --git a/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb b/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb
@@ -8,16 +8,44 @@
   describe '#perform' do
     let(:form_submission) { build(:debts_api_form5655_submission) }
 
-    context 'when all retries are exhausted' do
-      before do
-        allow(DebtsApi::V0::Form5655Submission).to receive(:find).and_return(form_submission)
-      end
+    before do
+      allow(DebtsApi::V0::Form5655Submission).to receive(:find).and_return(form_submission)
+    end
 
+    context 'when all retries are exhausted' do
       it 'sets submission to failure' do
         described_class.within_sidekiq_retries_exhausted_block({ 'jid' => 123 }) do
           expect(form_submission).to receive(:register_failure)
         end
       end
     end
+
+    context 'with retries exhausted' do
+      let(:config) { described_class }
+      let(:msg) do
+        {
+          'class' => 'YourJobClassName',
+          'args' => %w[123],
+          'jid' => '12345abcde',
+          'retry_count' => 5
+        }
+      end
+
+      let(:standard_exception) do
+        e = StandardError.new('abc-123')
+        allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2])
+        e
+      end
+
+      it 'increments the retries exhausted counter' do
+        statsd_key = DebtsApi::V0::Form5655::VHA::SharepointSubmissionJob::STATS_KEY
+
+        ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted", 'api.fsr_submission.failure'].each do |key|
+          expect(StatsD).to receive(:increment).with(key)
+        end
+
+        config.sidekiq_retries_exhausted_block.call(msg, standard_exception)
+      end
+    end
   end
 end
diff --git a/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb b/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb
@@ -9,6 +9,14 @@
     let(:form_submission) { build(:debts_api_form5655_submission) }
     let(:user) { build(:user, :loa3) }
     let(:user_data) { build(:user_profile_attributes) }
+    let(:msg) do
+      {
+        'class' => 'YourJobClassName',
+        'args' => %w[123 123-abc],
+        'jid' => '12345abcde',
+        'retry_count' => 5
+      }
+    end
 
     context 'when all retries are exhausted' do
       before do
@@ -20,6 +28,16 @@
           expect(form_submission).to receive(:register_failure)
         end
       end
+
+      it 'increments the retries exhausted counter' do
+        statsd_key = DebtsApi::V0::Form5655::VHA::VBSSubmissionJob::STATS_KEY
+
+        ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted", 'api.fsr_submission.failure'].each do |key|
+          expect(StatsD).to receive(:increment).with(key)
+        end
+
+        described_class.sidekiq_retries_exhausted_block.call(msg, StandardError.new('abc-123'))
+      end
     end
   end
 end
diff --git a/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb b/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb
@@ -123,9 +123,10 @@
           Backtrace: #{exception.backtrace.join("\n")}
         LOG
 
-        expect(StatsD).to receive(:increment).with(
-          "#{CopayNotifications::NewStatementNotificationJob::STATSD_KEY_PREFIX}.failure"
-        )
+        statsd_key = CopayNotifications::NewStatementNotificationJob::STATSD_KEY_PREFIX
+        ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted"].each do |key|
+          expect(StatsD).to receive(:increment).with(key)
+        end
         expect(Rails.logger).to receive(:error).with(expected_log_message)
         config.sidekiq_retries_exhausted_block.call(msg, exception)
       end

diff --git a/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb b/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb
@@ -54,5 +54,37 @@
         job.perform(statements_json_byte)
       end
     end
+
+    context 'with retries exhausted' do
+      let(:config) { described_class }
+      let(:exception) do
+        e = StandardError.new('Test error')
+        allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2])
+        e
+      end
+      let(:msg) do
+        {
+          'class' => 'YourJobClassName',
+          'args' => [statement],
+          'jid' => '12345abcde',
+          'retry_count' => 5
+        }
+      end
+
+      it 'logs the error' do
+        expected_log_message = <<~LOG
+          CopayNotifications::ParseNewStatementsJob retries exhausted:
+          Exception: #{exception.class} - #{exception.message}
+          Backtrace: #{exception.backtrace.join("\n")}
+        LOG
+
+        expect(StatsD).to receive(:increment).with(
+          "#{CopayNotifications::ParseNewStatementsJob::STATSD_KEY_PREFIX}.retries_exhausted"
+        )
+
+        expect(Rails.logger).to receive(:error).with(expected_log_message)
+        config.sidekiq_retries_exhausted_block.call(statements_json_byte, exception)
+      end
+    end
   end
 end