Merge branch 'burial-94168-remove-structureddata-processdatajob' of g…

…ithub.com:department-of-veterans-affairs/vets-api into burial-94168-remove-structureddata-processdatajob

.github/CODEOWNERS

@@ -594,6 +594,7 @@ app/uploaders/uploader_virus_scan.rb @department-of-veterans-affairs/va-api-engi
 app/uploaders/validate_pdf.rb @department-of-veterans-affairs/Disability-Experience @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/uploaders/vets_shrine.rb @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/validators/token_util.rb @department-of-veterans-affairs/backend-review-group
+app/uploaders/simple_forms_api/ @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/uploaders/veteran_facing_forms_remediation_uploader.rb @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/account_login_statistics_job.rb @department-of-veterans-affairs/octo-identity
 app/sidekiq/benefits_intake_remediation_status_job.rb @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -645,6 +646,7 @@ app/sidekiq/lighthouse/create_intent_to_file_job.rb @department-of-veterans-affa
 app/sidekiq/lighthouse/income_and_assets_intake_job.rb @department-of-veterans-affairs/pensions @department-of-veterans-affairs/backend-review-group
 app/sidekiq/load_average_days_for_claim_completion_job.rb @department-of-veterans-affairs/benefits-microservices @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/mhv @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+app/sidekiq/mhv/account_creator_job.rb @department-of-veterans-affairs/octo-identity
 app/sidekiq/pager_duty @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/preneeds @department-of-veterans-affairs/mbs-core-team @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 app/sidekiq/schema_contract @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -978,6 +980,7 @@ lib/sidekiq/semantic_logging.rb @department-of-veterans-affairs/backend-review-g
 lib/sidekiq/set_request_attributes.rb @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 lib/sidekiq/set_request_id.rb @department-of-veterans-affairs/backend-review-group @department-of-veterans-affairs/va-api-engineers
 lib/sign_in @department-of-veterans-affairs/octo-identity
+lib/simple_forms_api/ @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/simple_forms_api_submission @department-of-veterans-affairs/platform-va-product-forms @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/slack @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 lib/sm @department-of-veterans-affairs/vfs-mhv-secure-messaging @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
@@ -1341,6 +1344,7 @@ spec/sidekiq/kms_key_rotation @department-of-veterans-affairs/va-api-engineers @
 spec/sidekiq/lighthouse @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/load_average_days_for_claim_completion_job_spec.rb @department-of-veterans-affairs/benefits-microservices @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/mhv @department-of-veterans-affairs/vfs-mhv-medical-records @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
+spec/sidekiq/mhv/account_creator_job_spec.rb @department-of-veterans-affairs/octo-identity
 spec/sidekiq/pager_duty @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/preneeds @department-of-veterans-affairs/mbs-core-team @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group
 spec/sidekiq/schema_contract @department-of-veterans-affairs/va-api-engineers @department-of-veterans-affairs/backend-review-group

app/sidekiq/copay_notifications/new_statement_notification_job.rb

@@ -29,7 +29,8 @@ class NewStatementNotificationJob
     end
 
     sidekiq_retries_exhausted do |_msg, ex|
-      StatsD.increment("#{STATSD_KEY_PREFIX}.failure")
+      StatsD.increment("#{STATSD_KEY_PREFIX}.failure") # remove when we get more data into the retries_exhausted below
+      StatsD.increment("#{STATSD_KEY_PREFIX}.retries_exhausted")
       Rails.logger.error <<~LOG
         NewStatementNotificationJob retries exhausted:
         Exception: #{ex.class} - #{ex.message}

app/sidekiq/copay_notifications/parse_new_statements_job.rb

@@ -10,9 +10,19 @@ class ParseNewStatementsJob
     JOB_INTERVAL = Settings.mcp.notifications.job_interval
     # number of jobs to perform at next interval
     BATCH_SIZE = Settings.mcp.notifications.batch_size
+    STATSD_KEY_PREFIX = 'api.copay_notifications.json_file'
+
+    sidekiq_retries_exhausted do |_msg, ex|
+      StatsD.increment("#{STATSD_KEY_PREFIX}.retries_exhausted")
+      Rails.logger.error <<~LOG
+        CopayNotifications::ParseNewStatementsJob retries exhausted:
+        Exception: #{ex.class} - #{ex.message}
+        Backtrace: #{ex.backtrace.join("\n")}
+      LOG
+    end
 
     def perform(statements_json_byte)
-      StatsD.increment('api.copay_notifications.json_file.total')
+      StatsD.increment("#{STATSD_KEY_PREFIX}.total")
       # Decode and parse large json file (~60-90k objects)
       statements_json = Oj.load(Base64.decode64(statements_json_byte))
       unique_statements = statements_json.uniq { |statement| statement['veteranIdentifier'] }

app/sidekiq/mhv/account_creator_job.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'mhv/user_account/creator'
+
+module MHV
+  class AccountCreatorJob
+    include Sidekiq::Job
+
+    sidekiq_options retry: false
+
+    def perform(id)
+      user_verification = UserVerification.find(id)
+      MHV::UserAccount::Creator.new(user_verification:, break_cache: true).perform
+    rescue ActiveRecord::RecordNotFound
+      Rails.logger.error("MHV AccountCreatorJob failed: UserVerification not found for id #{id}")
+    end
+  end
+end

config/features.yml

@@ -794,6 +794,10 @@ features:
     actor_type: user
     description: Enable/disable dependent claimant support for POA requests
     enable_in_development: true
+  lighthouse_claims_api_v2_poa_va_notify:
+    actor_type: user
+    description: Enable/disable the VA ntofication emails in V2 POA
+    enable_in_development: false
   lighthouse_claims_v2_poa_requests_skip_bgs:
     actor_type: user
     description: Enable/disable skipping BGS calls for POA Requests

lib/evss/disability_compensation_form/form526_to_lighthouse_transform.rb

@@ -287,9 +287,9 @@ def transform_toxic_exposure(toxic_exposure_source) # rubocop:disable Metrics/Me
                                                              MULTIPLE_EXPOSURES_TYPE[:herbicide])
         end
 
-        if values_present(toxic_exposure_source['otherHerbicideLocations'])
+        if values_present(other_herbicide_locations) && other_herbicide_locations['description'].present?
           multiple_exposures +=
-            transform_multiple_exposures_other_details(toxic_exposure_source['otherHerbicideLocations'],
+            transform_multiple_exposures_other_details(other_herbicide_locations,
                                                        MULTIPLE_EXPOSURES_TYPE[:herbicide])
         end
 
@@ -300,9 +300,9 @@ def transform_toxic_exposure(toxic_exposure_source) # rubocop:disable Metrics/Me
                                                              MULTIPLE_EXPOSURES_TYPE[:hazard])
         end
 
-        if values_present(toxic_exposure_source['specifyOtherExposures'])
+        if values_present(specify_other_exposures) && specify_other_exposures['description'].present?
           multiple_exposures +=
-            transform_multiple_exposures_other_details(toxic_exposure_source['specifyOtherExposures'],
+            transform_multiple_exposures_other_details(specify_other_exposures,
                                                        MULTIPLE_EXPOSURES_TYPE[:hazard])
         end
 
@@ -410,7 +410,7 @@ def transform_gulf_war(gulf_war1990, gulf_war2001)
       def transform_herbicide(herbicide, other_herbicide_locations)
         filtered_results_herbicide = herbicide&.filter { |k| k != 'notsure' }
         herbicide_value = (values_present(filtered_results_herbicide) ||
-                          values_present(other_herbicide_locations)) &&
+          (other_herbicide_locations.present? && other_herbicide_locations['description'].present?)) &&
                           !none_of_these(filtered_results_herbicide)
 
         herbicide_service = Requests::HerbicideHazardService.new
@@ -420,7 +420,10 @@ def transform_herbicide(herbicide, other_herbicide_locations)
       end
 
       def transform_other_exposures(other_exposures, specify_other_exposures)
-        return nil if none_of_these(other_exposures) && !values_present(specify_other_exposures)
+        if none_of_these(other_exposures) &&
+           (specify_other_exposures.present? && specify_other_exposures['description'].blank?)
+          return nil
+        end
 
         filtered_results_other_exposures = other_exposures&.filter { |k, v| k != 'notsure' && v }
         additional_hazard_exposures_service = Requests::AdditionalHazardExposures.new
@@ -429,7 +432,9 @@ def transform_other_exposures(other_exposures, specify_other_exposures)
             HAZARDS_LH_ENUM[k.to_sym]
           end
         end
-        other = HAZARDS_LH_ENUM[:other] if values_present(specify_other_exposures)
+        if specify_other_exposures.present? && specify_other_exposures['description'].present?
+          other = HAZARDS_LH_ENUM[:other]
+        end
         additional_hazard_exposures_service.additional_exposures << other if other.present?
         return nil if additional_hazard_exposures_service.additional_exposures == []
 
@@ -743,6 +748,7 @@ def convert_date_no_day(date)
 
         # somehow, partial dates with the 'XX' (i.e. "2020-01-XX or 2020-XX-XX") are getting past FE validation
         # fix here in the backend while a proper FE solution is found
+        return nil if year.downcase.include?('x')
         return year if month.blank? || month.upcase == 'XX'
         return "#{year}-#{month}" if day.blank? || day.upcase == 'XX'
 

lib/simple_forms_api/form_remediation/configuration/base.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module SimpleFormsApi
+  module FormRemediation
+    module Configuration
+      class Base
+        attr_reader :id_type, :include_manifest, :include_metadata, :parent_dir, :presign_s3_url
+
+        def initialize
+          @id_type = :benefits_intake_uuid  # The field to query the FormSubmission by
+          @include_manifest = true          # Include a CSV file containing manifest data
+          @include_metadata = false         # Include a JSON file containing form submission metadata
+          @parent_dir = '/'                 # The base directory in the S3 bucket where the archive will be stored
+          @presign_s3_url = true            # Once archived to S3, the service should generate & return a presigned_url
+        end
+
+        # Override to inject your team's own submission archive
+        def submission_archive_class
+          SimpleFormsApi::FormRemediation::SubmissionArchive
+        end
+
+        # Override to inject your team's own s3 client
+        def s3_client
+          SimpleFormsApi::FormRemediation::S3Client
+        end
+
+        # Override to inject your team's own submission data builder
+        def remediation_data_class
+          SimpleFormsApi::FormRemediation::SubmissionRemediationData
+        end
+
+        # Override to inject your team's own file uploader
+        # If overriding this, s3_settings method doesn't have to be set
+        def uploader_class
+          SimpleFormsApi::FormRemediation::Uploader
+        end
+
+        # The FormSubmission model to query against
+        def submission_type
+          FormSubmission
+        end
+
+        # The attachment model to query for form submission attachments
+        def attachment_type
+          PersistentAttachment
+        end
+
+        # The temporary directory where form submissions will be
+        # hydrated and stored. This directory will automatically
+        # be deleted once the archive process completes
+        def temp_directory_path
+          @temp_directory_path ||= Rails.root.join("tmp/#{SecureRandom.hex}-archive/").to_s
+        end
+
+        # Used in the SimpleFormsApi::FormRemediation::Uploader S3 uploader
+        def s3_settings
+          raise NotImplementedError, 'Class must implement s3_settings method'
+        end
+
+        # The base S3 resource used for all S3 manipulations
+        def s3_resource
+          @s3_resource ||= uploader.new_s3_resource
+        end
+
+        # The bucket where payloads will be uploaded on S3
+        def target_bucket
+          @target_bucket ||= uploader.s3_bucket
+        end
+
+        # Utility method, override to add your own team's preferred logging approach
+        def log_info(message, **details)
+          Rails.logger.info(message, details)
+        end
+
+        # Utility method, override to add your own team's preferred logging approach
+        def log_error(message, error, **details)
+          Rails.logger.error(message, details.merge(error: error.message, backtrace: error.backtrace.first(5)))
+        end
+
+        # Utility method, override to add your own team's preferred logging approach
+        def handle_error(message, error, **details)
+          log_error(message, error, **details)
+          raise error
+        end
+      end
+    end
+  end
+end

lib/simple_forms_api/form_remediation/configuration/vff_config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'simple_forms_api/form_remediation/configuration/base'
+
+module SimpleFormsApi
+  module FormRemediation
+    module Configuration
+      class VffConfig < Base
+        def s3_settings
+          Settings.vff_simple_forms.aws
+        end
+      end
+    end
+  end
+end

modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/base_controller.rb

@@ -43,25 +43,52 @@ def status
         private
 
         def shared_form_validation(form_number)
+          # validate target veteran exists
+          target_veteran
+
           base = form_number == '2122' ? 'serviceOrganization' : 'representative'
           poa_code = form_attributes.dig(base, 'poaCode')
 
-          # Custom validations for POA submission, we must check this first
           @claims_api_forms_validation_errors = validate_form_2122_and_2122a_submission_values(
-            target_veteran.participant_id, user_profile, poa_code
+            user_profile:, veteran_participant_id: target_veteran.participant_id, poa_code:, base:
           )
-          # JSON validations for POA submission, will combine with previously captured errors and raise
+
           validate_json_schema(form_number.upcase)
           @rep_id = validate_registration_number!(base, poa_code)
 
           add_claimant_data_to_form if user_profile
-          # if we get here there were only validations file errors
+
           if @claims_api_forms_validation_errors
             raise ::ClaimsApi::Common::Exceptions::Lighthouse::JsonFormValidationError,
                   @claims_api_forms_validation_errors
           end
         end
 
+        def feature_enabled_and_claimant_present?
+          Flipper.enabled?(:lighthouse_claims_api_poa_dependent_claimants) && form_attributes['claimant'].present?
+        end
+
+        def assign_poa_to_dependent_claimant!(poa_code:)
+          return nil unless feature_enabled_and_claimant_present?
+
+          # Assign the veteranʼs file number
+          file_number_check
+
+          claimant = user_profile.profile
+
+          service = ClaimsApi::DependentClaimantPoaAssignmentService.new(
+            poa_code:,
+            veteran_participant_id: target_veteran.participant_id,
+            dependent_participant_id: claimant.participant_id,
+            veteran_file_number: @file_number,
+            allow_poa_access: form_attributes[:recordConsent].present? ? 'Y' : nil,
+            allow_poa_cadd: form_attributes[:consentAddressChange].present? ? 'Y' : nil,
+            claimant_ssn: claimant.ssn
+          )
+
+          service.assign_poa_to_dependent!
+        end
+
         def validate_registration_number!(base, poa_code)
           rn = form_attributes.dig(base, 'registrationNumber')
           rep = ::Veteran::Service::Representative.where('? = ANY(poa_codes) AND representative_id = ?',
@@ -193,8 +220,6 @@ def nullable_icn
         end
 
         def user_profile
-          return @user_profile if defined? @user_profile
-
           @user_profile ||= fetch_claimant
         end
 

modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/individual_controller.rb

@@ -16,6 +16,7 @@ def submit
           validate_veteran_name(false)
           poa_code = get_poa_code(FORM_NUMBER)
           validate_individual_poa_code!(poa_code)
+          assign_poa_to_dependent_claimant!(poa_code:)
 
           submit_power_of_attorney(poa_code, FORM_NUMBER)
         end

modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/organization_controller.rb

@@ -16,6 +16,7 @@ def submit
           validate_veteran_name(false)
           poa_code = get_poa_code(FORM_NUMBER)
           validate_org_poa_code!(poa_code)
+          assign_poa_to_dependent_claimant!(poa_code:)
 
           submit_power_of_attorney(poa_code, FORM_NUMBER)
         end

modules/claims_api/app/controllers/claims_api/v2/veterans/power_of_attorney/request_controller.rb

@@ -10,10 +10,11 @@ class PowerOfAttorney::RequestController < ClaimsApi::V2::Veterans::PowerOfAttor
         FORM_NUMBER = 'POA_REQUEST'
 
         def request_representative
+          # validate target veteran exists
+          target_veteran
+
           poa_code = form_attributes.dig('poa', 'poaCode')
-          @claims_api_forms_validation_errors = validate_form_2122_and_2122a_submission_values(
-            target_veteran.participant_id, user_profile, poa_code
-          )
+          @claims_api_forms_validation_errors = validate_form_2122_and_2122a_submission_values(user_profile:)
 
           validate_json_schema(FORM_NUMBER)
           validate_accredited_representative(form_attributes.dig('poa', 'registrationNumber'),