Support per-county low income thresholds (#485)

## Description

This has been hanging over our heads since Vermont, but now NY's HEAR
program has them, so it's time to actually do it.

I added a `type` field to each threshold definition, which can have
the values `hhsize` or `county-hhsize`, indicating the sequence of
keys to be used with the `thresholds` structure. Counties are
identified by FIPS code, same as in the authorities files.

To demonstrate this working, I updated the county-specific Vermont
incentives (it's a lot less data than the NY ones, which I'll do
separately).

I've also switched the low-income-thresholds JSON-Schema-to-TS logic
away from ajv and to `json-schema-to-ts`. The latter is what I prefer
to use for this because it's what Fastify uses to turn endpoint
schemas into types, and I think it's worthwhile to have everything use
the same solution for that (because every schema-to-TS thing has
subtle differences). I've been converting things gradually. Anyway,
this resulted most notably in having to remove the `required:
[1,2,...]` constraint on the household-size-thresholds struct, so I
added a unit test for that.

## Test Plan

New test for county-specific Vermont thresholds (same income is LI in
one county but not in another). Other tests pass.

scripts/incentive-spreadsheet-to-json.ts

@@ -9,7 +9,7 @@ import path from 'path';
 import { GaxiosPromise } from 'gaxios';
 import { GEO_GROUPS_BY_STATE, GeoGroupsByState } from '../src/data/geo_groups';
 import {
-  LOW_INCOME_THRESHOLDS_BY_AUTHORITY,
+  LOW_INCOME_THRESHOLDS_BY_STATE,
   LowIncomeThresholdsMap,
 } from '../src/data/low_income_thresholds';
 import {
@@ -186,7 +186,7 @@ async function convertToJson(
   lowIncome: boolean,
   geoGroups: boolean,
 ) {
-  if (lowIncome && !(state in LOW_INCOME_THRESHOLDS_BY_AUTHORITY)) {
+  if (lowIncome && !(state in LOW_INCOME_THRESHOLDS_BY_STATE)) {
     throw new Error(
       `No low-income thresholds defined for ${state} - define them or turn off strict mode.`,
     );
@@ -225,7 +225,7 @@ async function convertToJson(
     state,
     rows,
     strict,
-    lowIncome ? LOW_INCOME_THRESHOLDS_BY_AUTHORITY : null,
+    lowIncome ? LOW_INCOME_THRESHOLDS_BY_STATE : null,
     geoGroups ? GEO_GROUPS_BY_STATE : null,
   );
 

scripts/spreadsheets-health.test.ts

@@ -4,7 +4,7 @@ import _ from 'lodash';
 import fetch from 'make-fetch-happen';
 import { test } from 'tap';
 import { GEO_GROUPS_BY_STATE } from '../src/data/geo_groups';
-import { LOW_INCOME_THRESHOLDS_BY_AUTHORITY } from '../src/data/low_income_thresholds';
+import { LOW_INCOME_THRESHOLDS_BY_STATE } from '../src/data/low_income_thresholds';
 import {
   PASS_THROUGH_FIELDS,
   StateIncentive,
@@ -44,7 +44,7 @@ test('registered spreadsheets are in sync with checked-in JSON files', async tap
         state,
         rows,
         false,
-        LOW_INCOME_THRESHOLDS_BY_AUTHORITY,
+        LOW_INCOME_THRESHOLDS_BY_STATE,
         GEO_GROUPS_BY_STATE,
       );
       const invalidCollectedPath = file.filepath.replace(

scripts/update-fixtures.sh

@@ -220,6 +220,28 @@ curl \
 &utility=vt-vermont-electric-cooperative" \
   | jq . > test/fixtures/v1-vt-05845-vec-ev-low-income.json
 
+curl \
+  "http://localhost:3000/api/v1/calculator\
+?zip=05753\
+&owner_status=homeowner\
+&household_income=60000\
+&tax_filing=single\
+&household_size=1\
+&authority_types=state\
+&items=heat_pump_water_heater" \
+  | jq . > test/fixtures/v1-vt-addison-co-low-income.json
+
+curl \
+  "http://localhost:3000/api/v1/calculator\
+?zip=05201\
+&owner_status=homeowner\
+&household_income=60000\
+&tax_filing=single\
+&household_size=1\
+&authority_types=state\
+&items=heat_pump_water_heater" \
+  | jq . > test/fixtures/v1-vt-bennington-co-not-low-income.json
+
 # TODO: Remove beta states argument when CO is fully launched.
 curl \
   "http://localhost:3000/api/v1/calculator\

src/data/low_income_thresholds.ts

@@ -1,23 +1,10 @@
-import { JSONSchemaType } from 'ajv';
 import fs from 'fs';
+import { FromSchema } from 'json-schema-to-ts';
 
-export type LowIncomeThresholdsMap = {
-  [state: string]: StateLowIncomeThresholds;
-};
-
-export type StateLowIncomeThresholds = {
-  [authority_name: string]: LowIncomeThresholdsAuthority;
-};
-
-export type LowIncomeThresholdsAuthority = {
-  source_url: string;
-  thresholds: LowIncomeThresholds;
-  incentives: string[];
-};
-
-export type LowIncomeThresholds = {
-  [hhSize: string]: number;
-};
+export enum LowIncomeThresholdsType {
+  HH_SIZE = 'hh-size',
+  COUNTY_AND_HH_SIZE = 'county-and-hh-size',
+}
 
 // Add custom state low income authorities here
 export enum COLowIncomeAuthority {
@@ -44,73 +31,95 @@ export enum RILowIncomeAuthority {
   ENERGY = 'ri-rhode-island-energy',
 }
 
-export const AUTHORITY_THRESHOLDS_SCHEMA: JSONSchemaType<LowIncomeThresholds> =
-  {
-    type: 'object',
-    required: ['1', '2', '3', '4', '5', '6', '7', '8'],
-    additionalProperties: {
-      type: 'number',
-    },
-  } as const;
+export const HHSIZE_THRESHOLDS_SCHEMA = {
+  type: 'object',
+  patternProperties: {
+    '^[1-9][0-9]*$': { type: 'number' },
+  },
+  additionalProperties: false,
+} as const;
 
-export const AUTHORITY_INFO_SCHEMA: JSONSchemaType<LowIncomeThresholdsAuthority> =
-  {
-    type: 'object',
-    properties: {
-      source_url: { type: 'string' },
-      thresholds: AUTHORITY_THRESHOLDS_SCHEMA,
-      incentives: {
-        type: 'array',
-        items: { type: 'string' },
-        minItems: 1,
-        uniqueItems: true,
-      },
+export const AUTHORITY_INFO_SCHEMA = {
+  type: 'object',
+  properties: {
+    source_url: { type: 'string' },
+    incentives: {
+      type: 'array',
+      items: { type: 'string' },
+      minItems: 1,
+      uniqueItems: true,
     },
-    required: ['source_url', 'thresholds', 'incentives'],
-  } as const;
-
-export const STATE_THRESHOLDS_SCHEMA: JSONSchemaType<StateLowIncomeThresholds> =
-  {
-    type: 'object',
-    required: [],
-    dependentSchemas: {
-      CO: {
-        required: Object.values(COLowIncomeAuthority),
+  },
+  required: ['source_url', 'incentives'],
+  oneOf: [
+    {
+      properties: {
+        type: {
+          type: 'string',
+          const: 'hhsize',
+        },
+        thresholds: HHSIZE_THRESHOLDS_SCHEMA,
       },
-      IL: {
-        required: Object.values(ILIncomeAuthority),
-      },
-      NV: {
-        required: Object.values(NVLowIncomeAuthority),
-      },
-      RI: {
-        required: Object.values(RILowIncomeAuthority),
+      required: ['type', 'thresholds'],
+    },
+    {
+      properties: {
+        type: {
+          type: 'string',
+          const: 'county-hhsize',
+        },
+        thresholds: {
+          type: 'object',
+          patternProperties: {
+            // Keys are county FIPS codes: 5-digit numbers.
+            '^\\d{5}$': HHSIZE_THRESHOLDS_SCHEMA,
+            // Allow "other" as a fallback.
+            '^other$': HHSIZE_THRESHOLDS_SCHEMA,
+          },
+          additionalProperties: false,
+        },
       },
+      required: ['type', 'thresholds'],
     },
-    additionalProperties: AUTHORITY_INFO_SCHEMA,
-  };
+  ],
+} as const;
 
-// Keep states in alphabetic order.
-export const SCHEMA: JSONSchemaType<LowIncomeThresholdsMap> = {
+export const STATE_THRESHOLDS_SCHEMA = {
+  type: 'object',
+  required: [],
+  dependentSchemas: {
+    CO: {
+      required: Object.values(COLowIncomeAuthority),
+    },
+    IL: {
+      required: Object.values(ILIncomeAuthority),
+    },
+    NV: {
+      required: Object.values(NVLowIncomeAuthority),
+    },
+    RI: {
+      required: Object.values(RILowIncomeAuthority),
+    },
+  },
+  additionalProperties: AUTHORITY_INFO_SCHEMA,
+} as const;
+
+export const SCHEMA = {
   type: 'object',
-  required: [
-    'AZ',
-    'CO',
-    'CT',
-    'GA',
-    'IL',
-    'MI',
-    'NV',
-    'NY',
-    'OR',
-    'PA',
-    'RI',
-    'VA',
-    'VT',
-    'WI',
-  ],
   additionalProperties: STATE_THRESHOLDS_SCHEMA,
-};
+} as const;
+
+export type LowIncomeThresholdsMap = FromSchema<typeof SCHEMA>;
+
+export type StateLowIncomeThresholds = FromSchema<
+  typeof STATE_THRESHOLDS_SCHEMA
+>;
+
+export type LowIncomeThresholdsAuthority = FromSchema<
+  typeof AUTHORITY_INFO_SCHEMA
+>;
+
+export type HHSizeThresholds = FromSchema<typeof HHSIZE_THRESHOLDS_SCHEMA>;
 
-export const LOW_INCOME_THRESHOLDS_BY_AUTHORITY: LowIncomeThresholdsMap =
+export const LOW_INCOME_THRESHOLDS_BY_STATE: LowIncomeThresholdsMap =
   JSON.parse(fs.readFileSync('./data/low_income_thresholds.json', 'utf-8'));

src/lib/low-income.ts

@@ -0,0 +1,31 @@
+import {
+  HHSizeThresholds,
+  LowIncomeThresholdsAuthority,
+} from '../data/low_income_thresholds';
+import { CalculateParams } from './incentives-calculation';
+import { ResolvedLocation } from './location';
+
+/**
+ * Chooses the right income threshold and determines whether the given income
+ * is below it. Which threshold is chosen is conditioned on household size, and,
+ * in some cases, location.
+ */
+export function isLowIncome(
+  { household_size, household_income }: CalculateParams,
+  thresholds: LowIncomeThresholdsAuthority,
+  location: ResolvedLocation,
+): boolean {
+  const bySize: HHSizeThresholds =
+    thresholds.type === 'hhsize'
+      ? thresholds.thresholds
+      : thresholds.thresholds[location.countyFips] ??
+        thresholds.thresholds['other'];
+  const threshold = bySize?.[household_size];
+
+  // The only way the threshold should be missing is if they are defined by
+  // county, the user's county doesn't have thresholds defined, and there's no
+  // "other" fallback. (All possible input HH sizes should be present in the
+  // data; this is enforced by a unit test.) If the threshold is missing, be
+  // conservative and return false (i.e. "not low income").
+  return typeof threshold === 'number' && household_income <= threshold;
+}

src/lib/state-incentives-calculation.ts

@@ -2,7 +2,7 @@ import { min } from 'lodash';
 import { AuthoritiesByType, AuthorityType } from '../data/authorities';
 import { DATA_PARTNERS_BY_STATE } from '../data/data_partners';
 import { GEO_GROUPS_BY_STATE } from '../data/geo_groups';
-import { LOW_INCOME_THRESHOLDS_BY_AUTHORITY } from '../data/low_income_thresholds';
+import { LOW_INCOME_THRESHOLDS_BY_STATE } from '../data/low_income_thresholds';
 import {
   INCENTIVE_RELATIONSHIPS_BY_STATE,
   IncentiveRelationships,
@@ -27,6 +27,7 @@ import {
 } from './incentive-relationship-calculation';
 import { CalculateParams, CalculatedIncentive } from './incentives-calculation';
 import { ResolvedLocation } from './location';
+import { isLowIncome } from './low-income';
 import { isStateIncluded } from './states';
 import { estimateStateTaxAmount } from './tax-brackets';
 
@@ -90,21 +91,18 @@ export function calculateStateIncentivesAndSavings(
       eligible = false;
     }
 
-    if (!LOW_INCOME_THRESHOLDS_BY_AUTHORITY[stateId]) {
-      console.log('No income thresholds defined for ', stateId);
-    }
-    const thresholds_map = LOW_INCOME_THRESHOLDS_BY_AUTHORITY[stateId];
-
-    if (typeof thresholds_map !== 'undefined') {
-      if (
-        item.low_income &&
-        thresholds_map[item.low_income] &&
-        request.household_income >
-          thresholds_map[item.low_income].thresholds[request.household_size]
-      ) {
-        {
-          eligible = false;
-        }
+    if (item.low_income) {
+      const thresholds =
+        LOW_INCOME_THRESHOLDS_BY_STATE[stateId]?.[item.low_income];
+      if (!thresholds) {
+        console.log(
+          `No income thresholds defined for ${item.low_income} in ${stateId}`,
+        );
+        // The incentive is income-qualified but we don't know the thresholds;
+        // be conservative and exclude it.
+        eligible = false;
+      } else if (!isLowIncome(request, thresholds, location)) {
+        eligible = false;
       }
     }
 

test/data/low_income_thresholds.test.ts

@@ -1,5 +1,5 @@
 import { test } from 'tap';
-import { LOW_INCOME_THRESHOLDS_BY_AUTHORITY } from '../../src/data/low_income_thresholds';
+import { LOW_INCOME_THRESHOLDS_BY_STATE } from '../../src/data/low_income_thresholds';
 import {
   STATE_INCENTIVES_BY_STATE,
   StateIncentive,
@@ -18,7 +18,7 @@ test('low-income thresholds are equivalent in JSON config and incentives', async
   const map = computeIdToIncentiveMap(
     Object.values(STATE_INCENTIVES_BY_STATE).flat(),
   );
-  Object.entries(LOW_INCOME_THRESHOLDS_BY_AUTHORITY).forEach(
+  Object.entries(LOW_INCOME_THRESHOLDS_BY_STATE).forEach(
     ([, stateThresholds]) => {
       for (const [identifier, thresholds] of Object.entries(stateThresholds)) {
         for (const incentiveId of thresholds.incentives) {
@@ -41,7 +41,7 @@ test('low-income thresholds are equivalent in JSON config and incentives', async
   )) {
     for (const incentive of stateIncentives) {
       if (incentive.low_income) {
-        const stateThresholds = LOW_INCOME_THRESHOLDS_BY_AUTHORITY[stateId];
+        const stateThresholds = LOW_INCOME_THRESHOLDS_BY_STATE[stateId];
         t.hasProp(stateThresholds, incentive.low_income);
         t.ok(
           stateThresholds[incentive.low_income].incentives.includes(
@@ -52,3 +52,22 @@ test('low-income thresholds are equivalent in JSON config and incentives', async
     }
   }
 });
+
+test('low-income thresholds have HH sizes 1-8', async t => {
+  const hasRequiredKeys = (obj: Record<string, unknown>) => {
+    const keys = Object.keys(obj);
+    return [1, 2, 3, 4, 5, 6, 7, 8].every(num => keys.includes(num.toString()));
+  };
+
+  for (const stateThresholds of Object.values(LOW_INCOME_THRESHOLDS_BY_STATE)) {
+    for (const thresholds of Object.values(stateThresholds)) {
+      if (thresholds.type === 'hhsize') {
+        t.ok(hasRequiredKeys(thresholds.thresholds));
+      } else {
+        for (const countyThresholds of Object.values(thresholds.thresholds)) {
+          t.ok(hasRequiredKeys(countyThresholds));
+        }
+      }
+    }
+  }
+});