diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1fdc6e07548..fa03833ae6a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -594,6 +594,7 @@ app/uploaders/uploader_virus_scan.rb @department-of-veterans-affairs/va-api-engi app/uploaders/validate_pdf.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/uploaders/vets_shrine.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/validators/token_util.rb @department-of-veterans-affairs/backend-review-group +app/uploaders/simple_forms_api/ @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/uploaders/veteran_facing_forms_remediation_uploader.rb @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/account_login_statistics_job.rb @department-of-veterans-affairs/octo-identity app/sidekiq/benefits_intake_remediation_status_job.rb @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @@ -645,6 +646,7 @@ app/sidekiq/lighthouse/create_intent_to_file_job.rb @department-of-veterans-affa app/sidekiq/lighthouse/income_and_assets_intake_job.rb @department-of-veterans-affairs/pensions @department-of-veterans-affairs/backend-review-group app/sidekiq/load_average_days_for_claim_completion_job.rb @department-of-veterans-affairs/benefits-microservices @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/mhv @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group +app/sidekiq/mhv/account_creator_job.rb @department-of-veterans-affairs/octo-identity app/sidekiq/pager_duty @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/preneeds @department-of-veterans-affairs/mbs-core-team @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group app/sidekiq/schema_contract @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @@ -978,6 +980,7 @@ lib/sidekiq/semantic_logging.rb @department-of-veterans-affairs/backend-review-g lib/sidekiq/set_request_attributes.rb @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers lib/sidekiq/set_request_id.rb @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers lib/sign_in @department-of-veterans-affairs/octo-identity +lib/simple_forms_api/ @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group lib/simple_forms_api_submission @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group lib/slack @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group lib/sm @department-of-veterans-affairs/vfs-mhv-secure-messaging @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group @@ -1341,6 +1344,7 @@ spec/sidekiq/kms_key_rotation @department-of-veterans-affairs/va-api-engineers @ spec/sidekiq/lighthouse @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group spec/sidekiq/load_average_days_for_claim_completion_job_spec.rb @department-of-veterans-affairs/benefits-microservices @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group spec/sidekiq/mhv @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group +spec/sidekiq/mhv/account_creator_job_spec.rb @department-of-veterans-affairs/octo-identity spec/sidekiq/pager_duty @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group spec/sidekiq/preneeds @department-of-veterans-affairs/mbs-core-team @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group spec/sidekiq/schema_contract @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group diff --git a/app/sidekiq/copay_notifications/new_statement_notification_job.rb b/app/sidekiq/copay_notifications/new_statement_notification_job.rb index 9f16aba27a7..550ae630d36 100644 --- a/app/sidekiq/copay_notifications/new_statement_notification_job.rb +++ b/app/sidekiq/copay_notifications/new_statement_notification_job.rb @@ -29,7 +29,8 @@ class NewStatementNotificationJob end sidekiq_retries_exhausted do |_msg, ex| - StatsD.increment("#{STATSD_KEY_PREFIX}.failure") + StatsD.increment("#{STATSD_KEY_PREFIX}.failure") # remove when we get more data into the retries_exhausted below + StatsD.increment("#{STATSD_KEY_PREFIX}.retries_exhausted") Rails.logger.error <<~LOG NewStatementNotificationJob retries exhausted: Exception: #{ex.class} - #{ex.message} diff --git a/app/sidekiq/copay_notifications/parse_new_statements_job.rb b/app/sidekiq/copay_notifications/parse_new_statements_job.rb index 3f2bf9c348c..8472eac849f 100644 --- a/app/sidekiq/copay_notifications/parse_new_statements_job.rb +++ b/app/sidekiq/copay_notifications/parse_new_statements_job.rb @@ -10,9 +10,19 @@ class ParseNewStatementsJob JOB_INTERVAL = Settings.mcp.notifications.job_interval # number of jobs to perform at next interval BATCH_SIZE = Settings.mcp.notifications.batch_size + STATSD_KEY_PREFIX = 'api.copay_notifications.json_file' + + sidekiq_retries_exhausted do |_msg, ex| + StatsD.increment("#{STATSD_KEY_PREFIX}.retries_exhausted") + Rails.logger.error <<~LOG + CopayNotifications::ParseNewStatementsJob retries exhausted: + Exception: #{ex.class} - #{ex.message} + Backtrace: #{ex.backtrace.join("\n")} + LOG + end def perform(statements_json_byte) - StatsD.increment('api.copay_notifications.json_file.total') + StatsD.increment("#{STATSD_KEY_PREFIX}.total") # Decode and parse large json file (~60-90k objects) statements_json = Oj.load(Base64.decode64(statements_json_byte)) unique_statements = statements_json.uniq { |statement| statement['veteranIdentifier'] } diff --git a/app/sidekiq/mhv/account_creator_job.rb b/app/sidekiq/mhv/account_creator_job.rb new file mode 100644 index 00000000000..99ce9335afc --- /dev/null +++ b/app/sidekiq/mhv/account_creator_job.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'mhv/user_account/creator' + +module MHV + class AccountCreatorJob + include Sidekiq::Job + + sidekiq_options retry: false + + def perform(id) + user_verification = UserVerification.find(id) + MHV::UserAccount::Creator.new(user_verification:, break_cache: true).perform + rescue ActiveRecord::RecordNotFound + Rails.logger.error("MHV AccountCreatorJob failed: UserVerification not found for id #{id}") + end + end +end diff --git a/config/features.yml b/config/features.yml index 621a08767c6..8ee0f4bbefd 100644 --- a/config/features.yml +++ b/config/features.yml @@ -794,6 +794,10 @@ features: actor_type: user description: Enable/disable dependent claimant support for POA requests enable_in_development: true + lighthouse_claims_api_v2_poa_va_notify: + actor_type: user + description: Enable/disable the VA ntofication emails in V2 POA + enable_in_development: false lighthouse_claims_v2_poa_requests_skip_bgs: actor_type: user description: Enable/disable skipping BGS calls for POA Requests diff --git a/lib/evss/disability_compensation_form/form526_to_lighthouse_transform.rb b/lib/evss/disability_compensation_form/form526_to_lighthouse_transform.rb index fb380e898b6..0efb67f36e2 100644 --- a/lib/evss/disability_compensation_form/form526_to_lighthouse_transform.rb +++ b/lib/evss/disability_compensation_form/form526_to_lighthouse_transform.rb @@ -287,9 +287,9 @@ def transform_toxic_exposure(toxic_exposure_source) # rubocop:disable Metrics/Me MULTIPLE_EXPOSURES_TYPE[:herbicide]) end - if values_present(toxic_exposure_source['otherHerbicideLocations']) + if values_present(other_herbicide_locations) && other_herbicide_locations['description'].present? multiple_exposures += - transform_multiple_exposures_other_details(toxic_exposure_source['otherHerbicideLocations'], + transform_multiple_exposures_other_details(other_herbicide_locations, MULTIPLE_EXPOSURES_TYPE[:herbicide]) end @@ -300,9 +300,9 @@ def transform_toxic_exposure(toxic_exposure_source) # rubocop:disable Metrics/Me MULTIPLE_EXPOSURES_TYPE[:hazard]) end - if values_present(toxic_exposure_source['specifyOtherExposures']) + if values_present(specify_other_exposures) && specify_other_exposures['description'].present? multiple_exposures += - transform_multiple_exposures_other_details(toxic_exposure_source['specifyOtherExposures'], + transform_multiple_exposures_other_details(specify_other_exposures, MULTIPLE_EXPOSURES_TYPE[:hazard]) end @@ -410,7 +410,7 @@ def transform_gulf_war(gulf_war1990, gulf_war2001) def transform_herbicide(herbicide, other_herbicide_locations) filtered_results_herbicide = herbicide&.filter { |k| k != 'notsure' } herbicide_value = (values_present(filtered_results_herbicide) || - values_present(other_herbicide_locations)) && + (other_herbicide_locations.present? && other_herbicide_locations['description'].present?)) && !none_of_these(filtered_results_herbicide) herbicide_service = Requests::HerbicideHazardService.new @@ -420,7 +420,10 @@ def transform_herbicide(herbicide, other_herbicide_locations) end def transform_other_exposures(other_exposures, specify_other_exposures) - return nil if none_of_these(other_exposures) && !values_present(specify_other_exposures) + if none_of_these(other_exposures) && + (specify_other_exposures.present? && specify_other_exposures['description'].blank?) + return nil + end filtered_results_other_exposures = other_exposures&.filter { |k, v| k != 'notsure' && v } additional_hazard_exposures_service = Requests::AdditionalHazardExposures.new @@ -429,7 +432,9 @@ def transform_other_exposures(other_exposures, specify_other_exposures) HAZARDS_LH_ENUM[k.to_sym] end end - other = HAZARDS_LH_ENUM[:other] if values_present(specify_other_exposures) + if specify_other_exposures.present? && specify_other_exposures['description'].present? + other = HAZARDS_LH_ENUM[:other] + end additional_hazard_exposures_service.additional_exposures << other if other.present? return nil if additional_hazard_exposures_service.additional_exposures == [] @@ -743,6 +748,7 @@ def convert_date_no_day(date) # somehow, partial dates with the 'XX' (i.e. "2020-01-XX or 2020-XX-XX") are getting past FE validation # fix here in the backend while a proper FE solution is found + return nil if year.downcase.include?('x') return year if month.blank? || month.upcase == 'XX' return "#{year}-#{month}" if day.blank? || day.upcase == 'XX' diff --git a/lib/simple_forms_api/form_remediation/configuration/base.rb b/lib/simple_forms_api/form_remediation/configuration/base.rb new file mode 100644 index 00000000000..f1505918829 --- /dev/null +++ b/lib/simple_forms_api/form_remediation/configuration/base.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module SimpleFormsApi + module FormRemediation + module Configuration + class Base + attr_reader :id_type, :include_manifest, :include_metadata, :parent_dir, :presign_s3_url + + def initialize + @id_type = :benefits_intake_uuid # The field to query the FormSubmission by + @include_manifest = true # Include a CSV file containing manifest data + @include_metadata = false # Include a JSON file containing form submission metadata + @parent_dir = '/' # The base directory in the S3 bucket where the archive will be stored + @presign_s3_url = true # Once archived to S3, the service should generate & return a presigned_url + end + + # Override to inject your team's own submission archive + def submission_archive_class + SimpleFormsApi::FormRemediation::SubmissionArchive + end + + # Override to inject your team's own s3 client + def s3_client + SimpleFormsApi::FormRemediation::S3Client + end + + # Override to inject your team's own submission data builder + def remediation_data_class + SimpleFormsApi::FormRemediation::SubmissionRemediationData + end + + # Override to inject your team's own file uploader + # If overriding this, s3_settings method doesn't have to be set + def uploader_class + SimpleFormsApi::FormRemediation::Uploader + end + + # The FormSubmission model to query against + def submission_type + FormSubmission + end + + # The attachment model to query for form submission attachments + def attachment_type + PersistentAttachment + end + + # The temporary directory where form submissions will be + # hydrated and stored. This directory will automatically + # be deleted once the archive process completes + def temp_directory_path + @temp_directory_path ||= Rails.root.join("tmp/#{SecureRandom.hex}-archive/").to_s + end + + # Used in the SimpleFormsApi::FormRemediation::Uploader S3 uploader + def s3_settings + raise NotImplementedError, 'Class must implement s3_settings method' + end + + # The base S3 resource used for all S3 manipulations + def s3_resource + @s3_resource ||= uploader.new_s3_resource + end + + # The bucket where payloads will be uploaded on S3 + def target_bucket + @target_bucket ||= uploader.s3_bucket + end + + # Utility method, override to add your own team's preferred logging approach + def log_info(message, **details) + Rails.logger.info(message, details) + end + + # Utility method, override to add your own team's preferred logging approach + def log_error(message, error, **details) + Rails.logger.error(message, details.merge(error: error.message, backtrace: error.backtrace.first(5))) + end + + # Utility method, override to add your own team's preferred logging approach + def handle_error(message, error, **details) + log_error(message, error, **details) + raise error + end + end + end + end +end diff --git a/lib/simple_forms_api/form_remediation/configuration/vff_config.rb b/lib/simple_forms_api/form_remediation/configuration/vff_config.rb new file mode 100644 index 00000000000..a1311ad1a7c --- /dev/null +++ b/lib/simple_forms_api/form_remediation/configuration/vff_config.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'simple_forms_api/form_remediation/configuration/base' + +module SimpleFormsApi + module FormRemediation + module Configuration + class VffConfig < Base + def s3_settings + Settings.vff_simple_forms.aws + end + end + end + end +end diff --git a/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/base_controller.rb b/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/base_controller.rb index 231897db9f7..adddc96b67c 100644 --- a/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/base_controller.rb +++ b/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/base_controller.rb @@ -43,25 +43,52 @@ def status private def shared_form_validation(form_number) + # validate target veteran exists + target_veteran + base = form_number == '2122' ? 'serviceOrganization' : 'representative' poa_code = form_attributes.dig(base, 'poaCode') - # Custom validations for POA submission, we must check this first @claims_api_forms_validation_errors = validate_form_2122_and_2122a_submission_values( - target_veteran.participant_id, user_profile, poa_code + user_profile:, veteran_participant_id: target_veteran.participant_id, poa_code:, base: ) - # JSON validations for POA submission, will combine with previously captured errors and raise + validate_json_schema(form_number.upcase) @rep_id = validate_registration_number!(base, poa_code) add_claimant_data_to_form if user_profile - # if we get here there were only validations file errors + if @claims_api_forms_validation_errors raise ::ClaimsApi::Common::Exceptions::Lighthouse::JsonFormValidationError, @claims_api_forms_validation_errors end end + def feature_enabled_and_claimant_present? + Flipper.enabled?(:lighthouse_claims_api_poa_dependent_claimants) && form_attributes['claimant'].present? + end + + def assign_poa_to_dependent_claimant!(poa_code:) + return nil unless feature_enabled_and_claimant_present? + + # Assign the veteranʼs file number + file_number_check + + claimant = user_profile.profile + + service = ClaimsApi::DependentClaimantPoaAssignmentService.new( + poa_code:, + veteran_participant_id: target_veteran.participant_id, + dependent_participant_id: claimant.participant_id, + veteran_file_number: @file_number, + allow_poa_access: form_attributes[:recordConsent].present? ? 'Y' : nil, + allow_poa_cadd: form_attributes[:consentAddressChange].present? ? 'Y' : nil, + claimant_ssn: claimant.ssn + ) + + service.assign_poa_to_dependent! + end + def validate_registration_number!(base, poa_code) rn = form_attributes.dig(base, 'registrationNumber') rep = ::Veteran::Service::Representative.where('? = ANY(poa_codes) AND representative_id = ?', @@ -193,8 +220,6 @@ def nullable_icn end def user_profile - return @user_profile if defined? @user_profile - @user_profile ||= fetch_claimant end diff --git a/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/individual_controller.rb b/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/individual_controller.rb index 308817387b4..919935e4188 100644 --- a/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/individual_controller.rb +++ b/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/individual_controller.rb @@ -16,6 +16,7 @@ def submit validate_veteran_name(false) poa_code = get_poa_code(FORM_NUMBER) validate_individual_poa_code!(poa_code) + assign_poa_to_dependent_claimant!(poa_code:) submit_power_of_attorney(poa_code, FORM_NUMBER) end diff --git a/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/organization_controller.rb b/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/organization_controller.rb index 1b12d177e59..38d51e4689b 100644 --- a/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/organization_controller.rb +++ b/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/organization_controller.rb @@ -16,6 +16,7 @@ def submit validate_veteran_name(false) poa_code = get_poa_code(FORM_NUMBER) validate_org_poa_code!(poa_code) + assign_poa_to_dependent_claimant!(poa_code:) submit_power_of_attorney(poa_code, FORM_NUMBER) end diff --git a/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/request_controller.rb b/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/request_controller.rb index 092ecb7d5d3..1bed3c12d0f 100644 --- a/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/request_controller.rb +++ b/modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/request_controller.rb @@ -10,10 +10,11 @@ class PowerOfAttorney::RequestController < ClaimsApi::V2::Veterans::PowerOfAttor FORM_NUMBER = 'POA_REQUEST' def request_representative + # validate target veteran exists + target_veteran + poa_code = form_attributes.dig('poa', 'poaCode') - @claims_api_forms_validation_errors = validate_form_2122_and_2122a_submission_values( - target_veteran.participant_id, user_profile, poa_code - ) + @claims_api_forms_validation_errors = validate_form_2122_and_2122a_submission_values(user_profile:) validate_json_schema(FORM_NUMBER) validate_accredited_representative(form_attributes.dig('poa', 'registrationNumber'), diff --git a/modules/claims_api/app/controllers/concerns/claims_api/v2/power_of_attorney_validation.rb b/modules/claims_api/app/controllers/concerns/claims_api/v2/power_of_attorney_validation.rb index 9defb298c25..38e51f96e96 100644 --- a/modules/claims_api/app/controllers/concerns/claims_api/v2/power_of_attorney_validation.rb +++ b/modules/claims_api/app/controllers/concerns/claims_api/v2/power_of_attorney_validation.rb @@ -5,21 +5,71 @@ module ClaimsApi module V2 module PowerOfAttorneyValidation - def validate_form_2122_and_2122a_submission_values(veteran_participant_id, user_profile, poa_code) - validate_claimant(veteran_participant_id, user_profile, poa_code) + def validate_form_2122_and_2122a_submission_values(user_profile:, veteran_participant_id: nil, poa_code: nil, + base: nil) + validate_claimant_fields(user_profile) + if [veteran_participant_id, user_profile, poa_code, base].all?(&:present?) + validate_dependent_claimant(veteran_participant_id:, user_profile:, poa_code:, base:) + end + # collect errors and pass back to the controller raise_error_collection if @errors end private - def validate_claimant(veteran_participant_id, user_profile, poa_code) + def feature_enabled_and_claimant_present? + Flipper.enabled?(:lighthouse_claims_api_poa_dependent_claimants) && form_attributes['claimant'].present? + end + + def validate_dependent_claimant(veteran_participant_id:, user_profile:, poa_code:, base:) + return nil unless feature_enabled_and_claimant_present? + + service = build_dependent_claimant_verification_service(veteran_participant_id:, user_profile:, + poa_code:) + + validate_claimant(service:, base:) + end + + def build_dependent_claimant_verification_service(veteran_participant_id:, user_profile:, poa_code:) + claimant = user_profile.profile + + ClaimsApi::DependentClaimantVerificationService.new(veteran_participant_id:, + claimant_first_name: claimant.given_names.first, + claimant_last_name: claimant.family_name, + claimant_participant_id: claimant.participant_id, + poa_code:) + end + + def validate_claimant(service:, base:) + validate_poa_code(service:, base:) + validate_dependent(service:) + end + + def validate_poa_code(service:, base:) + service.validate_poa_code_exists! + rescue ::Common::Exceptions::UnprocessableEntity + collect_error_messages( + source: "/#{base}/poaCode", + detail: ClaimsApi::DependentClaimantVerificationService::POA_CODE_NOT_FOUND_ERROR_MESSAGE + ) + end + + def validate_dependent(service:) + service.validate_dependent_by_participant_id! + rescue ::Common::Exceptions::UnprocessableEntity + collect_error_messages( + source: '/claimant/claimantId', + detail: ClaimsApi::DependentClaimantVerificationService::CLAIMANT_NOT_A_DEPENDENT_ERROR_MESSAGE + ) + end + + def validate_claimant_fields(user_profile) return if form_attributes['claimant'].blank? validate_claimant_id_included(user_profile) validate_address validate_relationship - validate_dependent_claimant(veteran_participant_id, user_profile, poa_code) end def validate_address @@ -96,38 +146,6 @@ def validate_relationship end end - # rubocop:disable Metrics/MethodLength - def validate_dependent_claimant(veteran_participant_id, user_profile, poa_code) - unless Flipper.enabled?(:lighthouse_claims_api_poa_dependent_claimants) && form_attributes['claimant'].present? - return - end - - claimant = user_profile.profile - service = ClaimsApi::DependentClaimantVerificationService.new(veteran_participant_id:, - claimant_first_name: claimant.given_names.first, - claimant_last_name: claimant.family_name, - claimant_participant_id: claimant.participant_id, - poa_code:) - begin - service.validate_poa_code_exists! - rescue ::Common::Exceptions::UnprocessableEntity - collect_error_messages( - source: '/poaCode', - detail: ClaimsApi::DependentClaimantVerificationService::POA_CODE_NOT_FOUND_ERROR_MESSAGE - ) - end - - begin - service.validate_dependent_by_participant_id! - rescue ::Common::Exceptions::UnprocessableEntity - collect_error_messages( - source: '/claimant/claimantId', - detail: ClaimsApi::DependentClaimantVerificationService::CLAIMANT_NOT_A_DEPENDENT_ERROR_MESSAGE - ) - end - end - # rubocop:enable Metrics/MethodLength - def validate_claimant_id_included(user_profile) claimant_icn = form_attributes.dig('claimant', 'claimantId') if (user_profile.blank? || user_profile&.status == :not_found) && claimant_icn diff --git a/modules/claims_api/app/sidekiq/claims_api/poa_updater.rb b/modules/claims_api/app/sidekiq/claims_api/poa_updater.rb index 7a4ce84efa8..1bea2f09256 100644 --- a/modules/claims_api/app/sidekiq/claims_api/poa_updater.rb +++ b/modules/claims_api/app/sidekiq/claims_api/poa_updater.rb @@ -47,7 +47,11 @@ def enable_vbms_access?(poa_form:) end def vanotify?(auth_headers, rep) - auth_headers.key?(ClaimsApi::V2::Veterans::PowerOfAttorney::BaseController::VA_NOTIFY_KEY) && rep.present? + if Flipper.enabled?(:lighthouse_claims_api_v2_poa_va_notify) + auth_headers.key?(ClaimsApi::V2::Veterans::PowerOfAttorney::BaseController::VA_NOTIFY_KEY) && rep.present? + else + false + end end end end diff --git a/modules/claims_api/spec/requests/v2/veterans/power_of_attorney/2122_spec.rb b/modules/claims_api/spec/requests/v2/veterans/power_of_attorney/2122_spec.rb index 4197b05d6f0..153f0071b09 100644 --- a/modules/claims_api/spec/requests/v2/veterans/power_of_attorney/2122_spec.rb +++ b/modules/claims_api/spec/requests/v2/veterans/power_of_attorney/2122_spec.rb @@ -64,6 +64,91 @@ expect(response).to have_http_status(:accepted) end end + + describe 'lighthouse_claims_api_poa_dependent_claimants feature' do + let(:request_body) do + Rails.root.join('modules', 'claims_api', 'spec', 'fixtures', 'v2', 'veterans', + 'power_of_attorney', '2122', 'valid.json').read + end + let(:user_profile) do + MPI::Responses::FindProfileResponse.new( + status: :ok, + profile: MPI::Models::MviProfile.new( + given_names: %w[Not Under], + family_name: 'Test', + participant_id: '123', + ssn: '123456789' + ) + ) + end + + let(:claimant_data) do + { + claimantId: '456', # dependentʼs ICN + address: { + addressLine1: '123 anystreet', + city: 'anytown', + stateCode: 'OR', + country: 'USA', + zipCode: '12345' + }, + relationship: 'Child' + } + end + + before do + allow_any_instance_of(ClaimsApi::V2::Veterans::PowerOfAttorney::BaseController) + .to receive(:user_profile).and_return(user_profile) + allow_any_instance_of(ClaimsApi::DependentClaimantVerificationService) + .to receive(:validate_poa_code_exists!).and_return(nil) + allow_any_instance_of(ClaimsApi::DependentClaimantVerificationService) + .to receive(:validate_dependent_by_participant_id!).and_return(nil) + end + + context 'when the lighthouse_claims_api_poa_dependent_claimants feature is enabled' do + before do + Flipper.enable(:lighthouse_claims_api_poa_dependent_claimants) + end + + context 'and the request includes a claimant' do + it 'calls assign_poa_to_dependent!' do + VCR.use_cassette('claims_api/mpi/find_candidate/valid_icn_full') do + mock_ccg(scopes) do |auth_header| + json = JSON.parse(request_body) + json['data']['attributes']['claimant'] = claimant_data + request_body = json.to_json + + expect_any_instance_of(ClaimsApi::DependentClaimantPoaAssignmentService) + .to receive(:assign_poa_to_dependent!) + + post appoint_organization_path, params: request_body, headers: auth_header + end + end + end + end + end + + context 'when the lighthouse_claims_api_poa_dependent_claimants feature is disabled' do + before do + Flipper.disable(:lighthouse_claims_api_poa_dependent_claimants) + end + + it 'does not call assign_poa_to_dependent!' do + VCR.use_cassette('claims_api/mpi/find_candidate/valid_icn_full') do + mock_ccg(scopes) do |auth_header| + json = JSON.parse(request_body) + json['data']['attributes']['claimant'] = claimant_data + request_body = json.to_json + + expect_any_instance_of(ClaimsApi::DependentClaimantPoaAssignmentService) + .not_to receive(:assign_poa_to_dependent!) + + post appoint_organization_path, params: request_body, headers: auth_header + end + end + end + end + end end context 'when not valid' do diff --git a/modules/claims_api/spec/requests/v2/veterans/power_of_attorney/2122a_spec.rb b/modules/claims_api/spec/requests/v2/veterans/power_of_attorney/2122a_spec.rb index 562f44d9b37..1e6a83182b4 100644 --- a/modules/claims_api/spec/requests/v2/veterans/power_of_attorney/2122a_spec.rb +++ b/modules/claims_api/spec/requests/v2/veterans/power_of_attorney/2122a_spec.rb @@ -123,6 +123,65 @@ end end end + + describe 'lighthouse_claims_api_poa_dependent_claimants feature' do + let(:user_profile) do + MPI::Responses::FindProfileResponse.new( + status: :ok, + profile: MPI::Models::MviProfile.new( + given_names: %w[Not Under], + family_name: 'Test', + participant_id: '123', + ssn: '123456789' + ) + ) + end + + before do + allow_any_instance_of(ClaimsApi::V2::Veterans::PowerOfAttorney::BaseController) + .to receive(:user_profile).and_return(user_profile) + allow_any_instance_of(ClaimsApi::DependentClaimantVerificationService) + .to receive(:validate_poa_code_exists!).and_return(nil) + allow_any_instance_of(ClaimsApi::DependentClaimantVerificationService) + .to receive(:validate_dependent_by_participant_id!).and_return(nil) + end + + context 'when the lighthouse_claims_api_poa_dependent_claimants feature is enabled' do + before do + Flipper.enable(:lighthouse_claims_api_poa_dependent_claimants) + end + + context 'and the request includes a claimant' do + it 'calls assign_poa_to_dependent!' do + VCR.use_cassette('claims_api/mpi/find_candidate/valid_icn_full') do + mock_ccg(scopes) do |auth_header| + expect_any_instance_of(ClaimsApi::DependentClaimantPoaAssignmentService) + .to receive(:assign_poa_to_dependent!) + + post appoint_individual_path, params: claimant_data.to_json, headers: auth_header + end + end + end + end + end + + context 'when the lighthouse_claims_api_poa_dependent_claimants feature is disabled' do + before do + Flipper.disable(:lighthouse_claims_api_poa_dependent_claimants) + end + + it 'does not call assign_poa_to_dependent!' do + VCR.use_cassette('claims_api/mpi/find_candidate/valid_icn_full') do + mock_ccg(scopes) do |auth_header| + expect_any_instance_of(ClaimsApi::DependentClaimantPoaAssignmentService) + .not_to receive(:assign_poa_to_dependent!) + + post appoint_individual_path, params: claimant_data.to_json, headers: auth_header + end + end + end + end + end end context 'when not valid' do diff --git a/modules/claims_api/spec/sidekiq/poa_updater_spec.rb b/modules/claims_api/spec/sidekiq/poa_updater_spec.rb index d54ac820fe3..4bbed181432 100644 --- a/modules/claims_api/spec/sidekiq/poa_updater_spec.rb +++ b/modules/claims_api/spec/sidekiq/poa_updater_spec.rb @@ -132,6 +132,21 @@ end end + context 'when the flipper is on' do + it 'does not send the vanotify job' do + Flipper.disable(:lighthouse_claims_api_v2_poa_va_notify) + + poa.auth_headers.merge!({ + header_key => 'this_value' + }) + poa.save! + + expect(ClaimsApi::VANotifyJob).not_to receive(:perform_async) + + subject.new.perform(poa.id, 'Rep Data') + end + end + context 'does not send the va notify job' do it 'when the rep is not present' do poa.auth_headers.merge!({ diff --git a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb index 115e042f481..50e939a174c 100644 --- a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb +++ b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb @@ -12,8 +12,16 @@ class V0::Form5655::VBASubmissionJob class MissingUserAttributesError < StandardError; end - sidekiq_retries_exhausted do |job, _ex| + sidekiq_retries_exhausted do |job, ex| + StatsD.increment("#{STATS_KEY}.retries_exhausted") + submission_id = job['args'][0] user_uuid = job['args'][1] + Rails.logger.error <<~LOG + V0::Form5655::VBASubmissionJob retries exhausted: + submission_id: #{submission_id} | user_id: #{user_uuid} + Exception: #{ex.class} - #{ex.message} + Backtrace: #{ex.backtrace.join("\n")} + LOG UserProfileAttributes.find(user_uuid)&.destroy end diff --git a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb index c4b8f65fbb4..2b2e8b89847 100644 --- a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb +++ b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb @@ -13,7 +13,15 @@ class V0::Form5655::VHASubmissionJob class MissingUserAttributesError < StandardError; end sidekiq_retries_exhausted do |job, _ex| + StatsD.increment("#{STATS_KEY}.retries_exhausted") + submission_id = job['args'][0] user_uuid = job['args'][1] + Rails.logger.error <<~LOG + V0::Form5655::VHASubmissionJob retries exhausted: + Exception: #{ex.class} - #{ex.message} + Backtrace: #{ex.backtrace.join("\n")} + submission_id: #{submission_id} | user_id: #{user_uuid} + LOG UserProfileAttributes.find(user_uuid)&.destroy end diff --git a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb index bb36971fa5e..dfbaebe2e9b 100644 --- a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb +++ b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb @@ -11,7 +11,8 @@ class V0::Form5655::VHA::SharepointSubmissionJob sidekiq_options retry: 4 sidekiq_retries_exhausted do |job, _ex| - StatsD.increment("#{STATS_KEY}.failure") + StatsD.increment("#{STATS_KEY}.failure") # Deprecate this in favor of exhausted naming convention below + StatsD.increment("#{STATS_KEY}.retries_exhausted") submission_id = job['args'][0] submission = DebtsApi::V0::Form5655Submission.find(submission_id) submission.register_failure("SharePoint Submission Failed: #{job['error_message']}.") diff --git a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb index 20387a2ca50..c93dba7c128 100644 --- a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb +++ b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb @@ -13,7 +13,8 @@ class V0::Form5655::VHA::VBSSubmissionJob class MissingUserAttributesError < StandardError; end sidekiq_retries_exhausted do |job, _ex| - StatsD.increment("#{STATS_KEY}.failure") + StatsD.increment("#{STATS_KEY}.failure") # Deprecate this in favor of exhausted naming convention below + StatsD.increment("#{STATS_KEY}.retries_exhausted") submission_id = job['args'][0] user_uuid = job['args'][1] UserProfileAttributes.find(user_uuid)&.destroy diff --git a/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb b/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb index 9e271180e53..656bb14b187 100644 --- a/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb +++ b/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb @@ -38,5 +38,61 @@ expect(form_submission.error_message).to eq('uhoh') end end + + context 'with retries exhausted' do + let(:config) { described_class } + let(:missing_attributes_exception) do + e = DebtsApi::V0::Form5655::VBASubmissionJob::MissingUserAttributesError.new('abc-123') + allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2]) + e + end + + let(:standard_exception) do + e = StandardError.new('abc-123') + allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2]) + e + end + + let(:msg) do + { + 'class' => 'YourJobClassName', + 'args' => %w[123 123-abc], + 'jid' => '12345abcde', + 'retry_count' => 5 + } + end + + it 'handles MissingUserAttributesError' do + expected_log_message = <<~LOG + V0::Form5655::VBASubmissionJob retries exhausted: + submission_id: 123 | user_id: 123-abc + Exception: #{missing_attributes_exception.class} - #{missing_attributes_exception.message} + Backtrace: #{missing_attributes_exception.backtrace.join("\n")} + LOG + + expect(StatsD).to receive(:increment).with( + "#{DebtsApi::V0::Form5655::VBASubmissionJob::STATS_KEY}.retries_exhausted" + ) + + expect(Rails.logger).to receive(:error).with(expected_log_message) + config.sidekiq_retries_exhausted_block.call(msg, missing_attributes_exception) + end + + it 'handles unexpected errors' do + expected_log_message = <<~LOG + V0::Form5655::VBASubmissionJob retries exhausted: + submission_id: 123 | user_id: 123-abc + Exception: #{standard_exception.class} - #{standard_exception.message} + Backtrace: #{standard_exception.backtrace.join("\n")} + LOG + + expect(StatsD).to receive(:increment).with( + "#{DebtsApi::V0::Form5655::VBASubmissionJob::STATS_KEY}.retries_exhausted" + ) + + expect(Rails.logger).to receive(:error).with(expected_log_message) + config.sidekiq_retries_exhausted_block.call(msg, standard_exception) + end + end end end diff --git a/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb b/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb index b2c404a61a5..88b3f17192c 100644 --- a/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb +++ b/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb @@ -8,16 +8,44 @@ describe '#perform' do let(:form_submission) { build(:debts_api_form5655_submission) } - context 'when all retries are exhausted' do - before do - allow(DebtsApi::V0::Form5655Submission).to receive(:find).and_return(form_submission) - end + before do + allow(DebtsApi::V0::Form5655Submission).to receive(:find).and_return(form_submission) + end + context 'when all retries are exhausted' do it 'sets submission to failure' do described_class.within_sidekiq_retries_exhausted_block({ 'jid' => 123 }) do expect(form_submission).to receive(:register_failure) end end end + + context 'with retries exhausted' do + let(:config) { described_class } + let(:msg) do + { + 'class' => 'YourJobClassName', + 'args' => %w[123], + 'jid' => '12345abcde', + 'retry_count' => 5 + } + end + + let(:standard_exception) do + e = StandardError.new('abc-123') + allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2]) + e + end + + it 'increments the retries exhausted counter' do + statsd_key = DebtsApi::V0::Form5655::VHA::SharepointSubmissionJob::STATS_KEY + + ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted", 'api.fsr_submission.failure'].each do |key| + expect(StatsD).to receive(:increment).with(key) + end + + config.sidekiq_retries_exhausted_block.call(msg, standard_exception) + end + end end end diff --git a/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb b/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb index 64b77094e45..40a9c8b47b9 100644 --- a/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb +++ b/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb @@ -9,6 +9,14 @@ let(:form_submission) { build(:debts_api_form5655_submission) } let(:user) { build(:user, :loa3) } let(:user_data) { build(:user_profile_attributes) } + let(:msg) do + { + 'class' => 'YourJobClassName', + 'args' => %w[123 123-abc], + 'jid' => '12345abcde', + 'retry_count' => 5 + } + end context 'when all retries are exhausted' do before do @@ -20,6 +28,16 @@ expect(form_submission).to receive(:register_failure) end end + + it 'increments the retries exhausted counter' do + statsd_key = DebtsApi::V0::Form5655::VHA::VBSSubmissionJob::STATS_KEY + + ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted", 'api.fsr_submission.failure'].each do |key| + expect(StatsD).to receive(:increment).with(key) + end + + described_class.sidekiq_retries_exhausted_block.call(msg, StandardError.new('abc-123')) + end end end end diff --git a/modules/simple_forms_api/app/controllers/simple_forms_api/v1/uploads_controller.rb b/modules/simple_forms_api/app/controllers/simple_forms_api/v1/uploads_controller.rb index 3ded3c99839..ba16931016e 100644 --- a/modules/simple_forms_api/app/controllers/simple_forms_api/v1/uploads_controller.rb +++ b/modules/simple_forms_api/app/controllers/simple_forms_api/v1/uploads_controller.rb @@ -90,7 +90,7 @@ def handle_210966_authenticated confirmation_number, expiration_date = intent_service.submit form.track_user_identity(confirmation_number) - if Flipper.enabled?(:simple_forms_email_confirmations) + if confirmation_number && Flipper.enabled?(:simple_forms_email_confirmations) send_confirmation_email(parsed_form_data, get_form_id, confirmation_number) end diff --git a/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/docs/README.md b/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/docs/README.md new file mode 100644 index 00000000000..16f0ecfe335 --- /dev/null +++ b/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/docs/README.md @@ -0,0 +1,237 @@ +# Form Remediation + +This solution is designed to remediate form submissions which have failed submission and are over two weeks old. It is composed of several Ruby on Rails service objects which interact with each other. + +The primary use-case for this solution is for form remediation. This process consists of the following: + +1. Accept a form submission identifier. +1. Generate an archive payload consisting of the original form submission data as well as remediation specific documentation: + 1. Hydrate the original form submission + 1. Hydrate any original attachments that were a part of this submission + 1. Generate a JSON file with the original metadata from the submission + 1. Generate a manifest file to be used in the remediation process +1. Upload the generated archive as a .zip file onto the configured S3 bucket +1. Optionally return a presigned URL for accessing this file + +This solution also provides a means for storing and retrieving a single .pdf copy of the originally submitted form. + +The following image depicts how this solution is architected: + +![Error Remediation Architecture](./error_remediation_architecture.png) + +--- + +## Table of Contents + +- [Getting Started](#getting-started) + - [Settings](#settings) + - [Configuration](#configuration) +- [Usage](#usage) +- [Extending Functionality](#extending-functionality) +- [AWS S3 Bucket Setup](#aws-s3-bucket-setup) + +--- + +## Getting Started + +This service has been built in such a way that almost all aspects of the workflow can be configured or extended, depending upon your team's needs. + +### Settings + +The AWS S3 `region` and `bucket` are the only AWS credentials which need to be present. The S3 upload process uses a relatively new approach of using `vets-api`'s account role to access AWS. DevOps logic exists which defaults the AWS account key and secret to that role's credentials. + +In order to use the provided [Veteran Facing Forms uploader](../../../../../../../app/uploaders/simple_forms_api/form_remediation/uploader.rb), the AWS credentials will need to be included in the following format: + +```yml + bucket: + region: +``` + +If your team's credentials are in a different format, the s3_settings method can be overridden to account for this: + +```ruby +# frozen_string_literal: true + +require 'simple_forms_api/form_remediation/configuration/base' + +class NewConfig < SimpleFormsApi::FormRemediation::Configuration::Base + def s3_settings + settings = Settings.. + OpenStruct.new(bucket: settings.your_bucket_name, region: settings.your_region_name) + end +end +``` + +To create a `Settings` entry, follow the [documentation provided by Platform](https://depo-platform-documentation.scrollhelp.site/developer-docs/settings). + +### Configuration + +Your team can configure the service as it currently exists with minimal code additions. + +1. Create a new configuration file, inheriting from the base configuration class. Ensure that at the very least, the `s3_settings` method has been implemented. + +```ruby +# frozen_string_literal: true + +require 'simple_forms_api/form_remediation/configuration/base' + +class NewConfig < SimpleFormsApi::FormRemediation::Configuration::Base + def s3_settings + Settings.. + end +end +``` + +It's also worth noting that this solution by default queries FormSubmission's by the `:benefits_intake_uuid` identifier by default, but that can be overridden within the configuration by setting the `id_type` attribute. + +--- + +## Usage + +The Veteran Facing Forms team currently calls the bulk processing job utilizing [a rake task](../../../../../../simple_forms_api/lib/tasks/archive_forms_by_uuid.rake). Your team may choose to call it a different way but this is how we handle it. + +### Bulk Processing + +For convenience, we've created a job which processes multiple form submissions at once by iterating through a collection of UUIDs. This batch processing can be handled in multiple ways. + +The service can be called with our existing job: + +```ruby +config = YourTeamsConfig.new +job = SimpleFormsApi::FormRemediation::ArchiveBatchProcessingJob.perform(ids: your_teams_ids, config:) +presigned_urls = job.upload(type: :remediation) +``` + +Or your own custom job: + +```ruby +config = YourTeamsConfig.new +job = YourTeamsUniqueJob.perform(ids: your_teams_ids, config:) +presigned_urls = job.upload(type: :remediation) +``` + +### Processing Individually + +If your team isn't concerned with bulk processing, the S3 client itself handles processing an individual id for remediation out of the box: + +```ruby +config = YourTeamsConfig.new +client = SimpleFormsApi::FormRemediation::S3Client.new(config:, id: your_teams_id) +client.upload +``` + +The client also supports backing up individual form submissions in their original PDF format. This can be accomplished by changing the `type` to `:submission` during client initialization. This option will hydrate the form submission PDF and upload it to the configured S3 bucket and optionally return a presigned URL which links to the PDF itself. The subsequent remediation documentation is not created given this option. + +```ruby +config = YourTeamsConfig.new +client = SimpleFormsApi::FormRemediation::S3Client.new(config:, id: your_teams_id, type: :submission) +client.upload +``` + +--- + +## Extending Functionality + +In addition to configuration of existing logic, if your team requires something other than what this service provides, each step of the process can be substituted or skipped with basic inheritance. + +1. Take note of what can be extended and how it's used in the [base configuration](../../../../../../../lib/simple_forms_api/form_remediation/configuration/base.rb). +1. Create a new class, optionally extending the existing one. +1. Update your team's configuration to include the newly created class. These classes include: + 1. `submission_archive_class` - Override to inject your team's own submission archive + 1. `s3_client` - Override to inject your team's own s3 client + 1. `remediation_data_class` - Override to inject your team's own submission data builder service + 1. `uploader_class` - Override to inject your team's own file uploader + 1. `submission_type` - The FormSubmission model to query against + 1. `attachment_type` - The attachment model to query for form submission attachments +1. Pass the new configiration in when calling the service. + +```ruby +# frozen_string_literal: true + +require 'simple_forms_api/form_remediation/configuration/base' + +class NewUploader < SimpleFormsApi::FormRemediation::Uploader + def size_range + (1.byte)...(50.megabytes) + end +end +``` + +```ruby +# frozen_string_literal: true + +require 'simple_forms_api/form_remediation/configuration/base' + +class NewConfig < SimpleFormsApi::FormRemediation::Configuration::Base + def s3_settings + Settings.. + end + + def uploader_class + NewUploader + end +end +``` + +```ruby +config = NewConfig.new +job = SimpleFormsApi::FormRemediation::ArchiveBatchProcessingJob.new +presigned_urls = job.perform(ids: benefits_intake_uuids, config:, type: :remediation) +``` + +--- + +## AWS S3 Bucket Setup + +If your team does not have their own dedicated AWS S3 bucket, the following steps will need to be taken. + +### S3 Bucket Request Process + +#### **Submit PR** + +- **Preferred Approach by Platform DevOps**: Create a new configuration PR to add the necessary S3 bucket(s) using these PRs as guidance (these changes can be done in a single PR): [Sample PR #1](https://github.com/department-of-veterans-affairs/devops/pull/14735), [Sample PR #2](https://github.com/department-of-veterans-affairs/devops/pull/14742) + - Include the appropriate configuration for both staging and production environments. + - Ensure you follow your team's naming convention when creating the bucket(s). + +#### **Review Process** + +- Once the PR is submitted, request a review from the DevOps/Platform team. + - A teammate should review the PR first before requesting a review from DevOps. +- Ask for a **sanity check** to ensure the bucket is provisioned correctly, securely, and consistently with existing infrastructure. + +#### **Apply Changes** + +- After the PR is merged, the DevOps team will apply the Terraform changes to provision the bucket(s) in the appropriate environment (staging/production). + - At the time of writing this document, this process is manual for DevOps and will need to be asked for explicitly once the PR has been merged. +- Once the bucket(s) are successfully created, the DevOps or platform team will provide confirmation. + +#### **Access and Credentials** + +**Production/Staging Environments**: + +- The `vets-api` service will automatically have access to the S3 bucket in production and staging environments through the **vets-api pod's service account**. +- No explicit AWS Access Key or Secret Key is needed in these environments, as the credentials will be [fetched automatically](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#initialize-instance_method) by the pod service account. + +**Local Development/Non-Production Environments**: + +- For local testing or other non-production environments, you **will need to pass your own AWS credentials** (e.g., via environment variables or AWS CLI) when working with S3. + +**Testing Credentials**: + +- Ensure that the `vets-api` role can write to the S3 bucket in both staging and production. You should test your S3 client with the `vets-api` role and report any errors. +- If you're using an AWS client (e.g., via the AWS SDK), it should automatically use the `vets-api` role in the target environment. + +**Documentation**: + +- Internal documentation on using the `vets-api` role with AWS clients should be consulted for further guidance. For local development, ensure AWS credentials are properly set up. + +### S3 Bucket Naming Convention + +- **For Staging**: `dsva-vagov-staging-[team-project-name]` +- **For Production**: `dsva-vagov-prod-[team-project-name]` + +### Guidelines for Future Infrastructure Requests + +- Teams should continue to request infrastructure changes via PRs to the DevOps repository. +- If you're unfamiliar with Terraform or any other infrastructure specific technologies, request assistance from the Platform or DevOps team to ensure the infrastructure is provisioned correctly. +- Ensure proper testing in both staging and production environments, verifying that the `vets-api` role has the appropriate permissions to interact with the S3 bucket. diff --git a/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/docs/error_remediation_architecture.png b/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/docs/error_remediation_architecture.png new file mode 100644 index 00000000000..010a3e5eeec Binary files /dev/null and b/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/docs/error_remediation_architecture.png differ diff --git a/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/file_utilities.rb b/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/file_utilities.rb new file mode 100644 index 00000000000..b329ab7242b --- /dev/null +++ b/modules/simple_forms_api/app/services/simple_forms_api/form_remediation/file_utilities.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'simple_forms_api/form_remediation/configuration/base' + +module SimpleFormsApi + module FormRemediation + module FileUtilities + def zip_directory!(parent_dir, file_path) + base_dir = build_path(:dir, parent_dir, 'remediation', ext: '.zip') + raise "Directory not found: #{base_dir}" unless File.directory?(base_dir) + + Zip::File.open(file_path, Zip::File::CREATE) do |zipfile| + Dir.chdir(base_dir) do + Dir['**', '*'].each do |file| + next if File.directory?(file) + + zipfile.add(file, File.join(base_dir, file)) if File.file?(file) + end + end + end + + file_path + rescue => e + handle_error("Failed to zip temp directory: #{base_dir} to location: #{file_path}", e) + end + + def cleanup(path) + FileUtils.rm_rf(path) + end + + def create_temp_directory!(dir_path) + FileUtils.mkdir_p(dir_path) + end + + def build_local_file_dir!(s3_key, dir_path, s3_dir_path) + local_path = Pathname.new(s3_key).relative_path_from(Pathname.new(s3_dir_path)) + final_path = Pathname.new(dir_path).join(local_path) + + FileUtils.mkdir_p(final_path.dirname) + final_path.to_s + end + + def build_path(path_type, base_dir, *, ext: '.pdf') + file_ext = path_type == :file ? ext : '' + path = Pathname.new(base_dir.to_s).join(*).sub_ext(file_ext) + path.to_s + end + + def write_file(dir_path, file_name, payload) + File.write(File.join(dir_path, file_name), payload) + end + + def unique_file_path(form_number, id) + [Time.zone.today.strftime('%-m.%d.%y'), 'form', form_number, 'vagov', id].join('_') + end + + private + + def handle_error(*, **) + config = Configuration::Base.new + config.handle_error(*, **) + end + end + end +end diff --git a/modules/simple_forms_api/spec/requests/simple_forms_api/v1/simple_forms_spec.rb b/modules/simple_forms_api/spec/requests/simple_forms_api/v1/simple_forms_spec.rb index dab7485ed1c..4f13488d605 100644 --- a/modules/simple_forms_api/spec/requests/simple_forms_api/v1/simple_forms_spec.rb +++ b/modules/simple_forms_api/spec/requests/simple_forms_api/v1/simple_forms_spec.rb @@ -878,6 +878,24 @@ expect(VANotify::EmailJob).not_to have_received(:perform_async) end end + + context 'no new intent to file added' do + before do + allow_any_instance_of(SimpleFormsApi::IntentToFile).to receive(:submit).and_return([nil, Time.zone.now]) + allow_any_instance_of(SimpleFormsApi::IntentToFile) + .to receive(:existing_intents) + .and_return({ 'compensation' => {}, 'pension' => {}, 'survivor' => {} }) + end + + it 'does not send a confirmation email' do + data['preparer_identification'] = 'VETERAN' + + post '/simple_forms_api/v1/simple_forms', params: data + + expect(response).to have_http_status(:ok) + expect(VANotify::EmailJob).not_to have_received(:perform_async) + end + end end end end diff --git a/spec/lib/evss/disability_compensation_form/form526_to_lighthouse_transform_spec.rb b/spec/lib/evss/disability_compensation_form/form526_to_lighthouse_transform_spec.rb index eb33b0f4e7a..512b56b33e6 100644 --- a/spec/lib/evss/disability_compensation_form/form526_to_lighthouse_transform_spec.rb +++ b/spec/lib/evss/disability_compensation_form/form526_to_lighthouse_transform_spec.rb @@ -96,6 +96,67 @@ end end + describe 'specifyOtherExposure and otherHerbicideLocations' do + let(:submission) { create(:form526_submission, :with_everything_toxic_exposure) } + let(:data) { submission.form['form526'] } + + context 'multiple exposures' do + it 'is processed if descriptions are present' do + # specifyOtherExposure and otherHerbicideLocations should be processed if descriptions are present + data['form526']['toxicExposure'] = { + 'gulfWar2001Details' => {}, + 'gulfWar1990Details' => {}, + 'herbicide' => {}, + 'herbicideDetails' => {}, + 'otherExposureDetails' => {}, + 'specifyOtherExposures' => { + 'description' => 'specifyOtherExposures', + 'startDate' => '1992-01-01', + 'endDate' => '1993-01-01' + }, + 'otherHerbicideLocations' => { + 'description' => 'otherHerbicideLocations', + 'startDate' => '1992-01-01', + 'endDate' => '1993-01-01' + } + } + result = transformer.transform(data) + + # "other" objects should have been processed for MultipleExposures because the descriptions are there + expect(result.toxic_exposure.multiple_exposures.length).to eq(2) + expect(result.toxic_exposure.multiple_exposures.first.exposure_location).to eq('otherHerbicideLocations') + expect(result.toxic_exposure.multiple_exposures.first.hazard_exposed_to).to eq(nil) + expect(result.toxic_exposure.multiple_exposures.last.exposure_location).to eq(nil) + expect(result.toxic_exposure.multiple_exposures.last.hazard_exposed_to).to eq('specifyOtherExposures') + end + + it 'is not processed if missing descriptions' do + # specifyOtherExposure and otherHerbicideLocations should not be processed if missing descriptions + data['form526']['toxicExposure'] = { + 'gulfWar2001Details' => {}, + 'gulfWar1990Details' => {}, + 'herbicide' => {}, + 'herbicideDetails' => {}, + 'otherExposureDetails' => {}, + 'specifyOtherExposures' => { + 'description' => ' ', + 'startDate' => '1992-01-01', + 'endDate' => '1993-01-01' + }, + 'otherHerbicideLocations' => { + 'description' => ' ', + 'startDate' => '1992-01-01', + 'endDate' => '1993-01-01' + } + } + result = transformer.transform(data) + # nothing should have been processed for MultipleExposures because + # all of the details are missing and the descriptions are missing in the "other" objects + expect(result.toxic_exposure.multiple_exposures).to eq([]) + end + end + end + describe '#evss_claims_process_type' do let(:submission) { create(:form526_submission, :with_everything) } let(:data) { submission.form['form526'] } @@ -534,6 +595,14 @@ date = '2024' result = transformer.send(:convert_date_no_day, date) expect(result).to eq('2024') + + date = 'XXXX' + result = transformer.send(:convert_date_no_day, date) + expect(result).to eq(nil) + + date = 'XX-XX-XX' + result = transformer.send(:convert_date_no_day, date) + expect(result).to eq(nil) end it 'set served_in_herbicide_hazard_locations correctly' do @@ -606,6 +675,31 @@ result = transformer.send(:transform_herbicide, other_herbicide_locations_has_blank_fields['herbicide'], other_herbicide_locations_has_blank_fields['otherHerbicideLocations']) expect(result.served_in_herbicide_hazard_locations).to eq('NO') + + # description must be in the otherHerbicideLocations object for it to be processed (blank string) + other_herbicide_locations_description_blank = data.merge({ + 'herbicide' => nil, + 'otherHerbicideLocations' => { + 'description' => '', + 'startDate' => '1992-01-01', + 'endDate' => '1992-01-01' + } + }) + result = transformer.send(:transform_herbicide, other_herbicide_locations_description_blank['herbicide'], + other_herbicide_locations_description_blank['otherHerbicideLocations']) + expect(result.served_in_herbicide_hazard_locations).to eq('NO') + + # description must be in the otherHerbicideLocations object for it to be processed (attribute missing) + other_herbicide_locations_description_missing = data.merge({ + 'herbicide' => nil, + 'otherHerbicideLocations' => { + 'startDate' => '1992-01-01', + 'endDate' => '1992-01-01' + } + }) + result = transformer.send(:transform_herbicide, other_herbicide_locations_description_missing['herbicide'], + other_herbicide_locations_description_missing['otherHerbicideLocations']) + expect(result.served_in_herbicide_hazard_locations).to eq('NO') end it 'set additional_hazard_exposures correctly' do @@ -709,6 +803,32 @@ result = transformer.send(:transform_other_exposures, none_option_with_no_other['otherExposures'], none_option_with_no_other['specifyOtherExposures']) expect(result).to eq(nil) + + # description must be in the specifyOtherExposures object for it to be processed (blank string) + no_option_with_no_other_blank_string = data.merge({ + 'otherExposures' => {}, + 'specifyOtherExposures' => { + 'description' => '', + 'startDate' => '1991-03-01', + 'endDate' => '1992-01-01' + } + }) + result = transformer.send(:transform_other_exposures, no_option_with_no_other_blank_string['otherExposures'], + no_option_with_no_other_blank_string['specifyOtherExposures']) + expect(result).to eq(nil) + + # description must be in the specifyOtherExposures object for it to be processed (missing attribute) + no_option_with_no_other_missing = data.merge({ + 'otherExposures' => {}, + 'specifyOtherExposures' => { + 'startDate' => '1991-03-01', + 'endDate' => '1992-01-01' + } + }) + result = transformer.send(:transform_other_exposures, no_option_with_no_other_missing['otherExposures'], + no_option_with_no_other_missing['specifyOtherExposures']) + + expect(result).to eq(nil) end it 'filters unselected items from details objects correctly' do diff --git a/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb b/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb index 5f704d3dbbc..5d5d24a119b 100644 --- a/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb +++ b/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb @@ -123,9 +123,10 @@ Backtrace: #{exception.backtrace.join("\n")} LOG - expect(StatsD).to receive(:increment).with( - "#{CopayNotifications::NewStatementNotificationJob::STATSD_KEY_PREFIX}.failure" - ) + statsd_key = CopayNotifications::NewStatementNotificationJob::STATSD_KEY_PREFIX + ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted"].each do |key| + expect(StatsD).to receive(:increment).with(key) + end expect(Rails.logger).to receive(:error).with(expected_log_message) config.sidekiq_retries_exhausted_block.call(msg, exception) end diff --git a/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb b/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb index 861006f3c26..01ab6ed52e3 100644 --- a/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb +++ b/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb @@ -54,5 +54,37 @@ job.perform(statements_json_byte) end end + + context 'with retries exhausted' do + let(:config) { described_class } + let(:exception) do + e = StandardError.new('Test error') + allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2]) + e + end + let(:msg) do + { + 'class' => 'YourJobClassName', + 'args' => [statement], + 'jid' => '12345abcde', + 'retry_count' => 5 + } + end + + it 'logs the error' do + expected_log_message = <<~LOG + CopayNotifications::ParseNewStatementsJob retries exhausted: + Exception: #{exception.class} - #{exception.message} + Backtrace: #{exception.backtrace.join("\n")} + LOG + + expect(StatsD).to receive(:increment).with( + "#{CopayNotifications::ParseNewStatementsJob::STATSD_KEY_PREFIX}.retries_exhausted" + ) + + expect(Rails.logger).to receive(:error).with(expected_log_message) + config.sidekiq_retries_exhausted_block.call(statements_json_byte, exception) + end + end end end diff --git a/spec/sidekiq/mhv/account_creator_job_spec.rb b/spec/sidekiq/mhv/account_creator_job_spec.rb new file mode 100644 index 00000000000..a51947cba3f --- /dev/null +++ b/spec/sidekiq/mhv/account_creator_job_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'sidekiq/testing' + +RSpec.describe MHV::AccountCreatorJob, type: :job do + let(:user_account) { create(:user_account, icn:) } + let(:user_verification) { create(:user_verification, user_account:, user_credential_email:) } + let(:user_credential_email) { create(:user_credential_email, credential_email: email) } + let!(:terms_of_use_agreement) { create(:terms_of_use_agreement, user_account:) } + let(:icn) { '10101V964144' } + let(:email) { 'some-email@email.com' } + let(:mhv_client) { instance_double(MHV::AccountCreation::Service) } + let(:job) { described_class.new } + let(:break_cache) { true } + let(:mhv_response_body) do + { + user_profile_id: '12345678', + premium: true, + champ_va: true, + patient: true, + sm_account_created: true + } + end + + before do + allow(MHV::AccountCreation::Service).to receive(:new).and_return(mhv_client) + allow(mhv_client).to receive(:create_account).and_return(mhv_response_body) + end + + describe '#perform' do + Sidekiq::Testing.inline! do + context 'when a UserVerification exists' do + it 'calls the MHV::UserAccount::Creator service class and returns the created MHVUserAccount instance' do + expect(MHV::UserAccount::Creator).to receive(:new).with(user_verification:, break_cache:).and_call_original + job.perform(user_verification.id) + end + + context 'when the MHV API call is successful' do + it 'creates & returns a new MHVUserAccount instance' do + response = job.perform(user_verification.id) + expect(response).to be_an_instance_of(MHVUserAccount) + end + end + end + + context 'when a UserVerification does not exist' do + let(:expected_error_id) { 999 } + let(:expected_error_message) do + "MHV AccountCreatorJob failed: UserVerification not found for id #{expected_error_id}" + end + + it 'logs an error' do + expect(Rails.logger).to receive(:error).with(expected_error_message) + job.perform(expected_error_id) + end + end + end + end +end