From f3772861d05367351cf371fbd65f75c195c1baee Mon Sep 17 00:00:00 2001 From: Adam Moussa Date: Tue, 21 Jul 2026 19:40:25 -0400 Subject: [PATCH] feat(fetchboa): v2 return/redeposit-aware reconciliation (#66) The two-code 475/255 filter missed every return on the Jan-Jul statement (29 ARP refer-to-maker credits, 24 electronic return credits, and the redeposit cycles), leaving 5 paid-then-returned checks stuck at Cleared ($5,256.62, vendors unpaid). Rewrite fetchBoaTransactions around a pure reconciliation module (src/boaRecon.js) covered by node:test: - BAI transaction-code event map (only 475 = check paid is confirmed; remaining codes pend the enumeration replay) with a description-text fallback classifier for ARP refer-to-maker, return-of-posted-check (check-numbered and electronic) and ACH DES:PAYMENTS formats. Unknown check-shaped transactions are logged and counted, never dropped. - Matching on check number AND amount over all candidates; wrong-amount or collapsed-number postings fall back to a unique exact-amount match within checks issued in the last 120 days; ambiguity means unmatched with no write. - Transitions always set BOTH status and clear_status (the missed returns slipped through the divergence between them). clear_status is bank truth: returns apply even to Cleared records, a second paid debit on a Returned check is a redeposit back to Cleared, and a return on a voided check is terminal voided-and-bounced. Every applied event is appended to a history list; identical replayed events are noops. - Optional {fromDate, toDate} replay payload (strictly validated) for gap replays and the BAI-code enumeration runs; default stays yesterday. - Each run writes a boa_recon# summary item (90-day TTL) with per-event counts, unmatched check numbers/amounts, and unknown codes; unmatched/unknown also console.error (Slack alerting is #71). ACH transactions are classified but not yet acted on; bank-confirming ACH lands with #69. --- README.md | 25 ++- package.json | 3 + src/boaRecon.js | 327 ++++++++++++++++++++++++++++++ src/fetchBoaTransactions.js | 193 ++++++++++++++---- tests/boaRecon.test.js | 394 ++++++++++++++++++++++++++++++++++++ 5 files changed, 897 insertions(+), 45 deletions(-) create mode 100644 src/boaRecon.js create mode 100644 tests/boaRecon.test.js diff --git a/README.md b/README.md index 3603522..4d3c71b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ AWS SAM application that ingests payment CSVs, syncs check data with Bank of Ame - **ProcessPayrollEmail** — Lambda triggered by S3 (inbound email) and SQS (batch timer). SES receives Gusto payroll emails at `payroll@int.seahaven.com`, stores them to S3, and this Lambda parses the email body, extracts financial data, and posts a combined Slack notification (employee payroll + contractor payments) after a 10-minute batching window. Runs outside VPC. - **ProcessPaymentCsv** — Lambda triggered by S3 CSV upload. Parses Stampli payment exports, upserts to DynamoDB, and submits new/cancelled checks to the CashPro Check Management API. -- **FetchBoaTransactions** — Scheduled Lambda (weekdays 9am ET). Calls the CashPro Previous Day Transaction Inquiry API and matches cleared/returned checks back to DynamoDB records. +- **FetchBoaTransactions** — Scheduled Lambda (weekdays 9am ET). Calls the CashPro Previous Day Transaction Inquiry API, classifies each transaction (paid checks, ARP refer-to-maker and return-of-posted-check credits, electronic returns), and reconciles them onto DynamoDB payment records. See [Bank reconciliation](#bank-reconciliation-fetchboatransactions). - **SlackAppHome** — Lambda behind API Gateway (`POST /slack/events`). Verifies the Slack signing secret (HMAC-SHA256, 5-minute replay window) before processing, then renders the payments dashboard on the Slack App Home tab with outstanding aging buckets and drill-down modals. - **ExpenseReceiver** — Lambda behind API Gateway (`POST /slack/expense-events`). Verifies the Slack signing secret (HMAC-SHA256), handles URL verification challenges, and async-invokes ExpenseProcessor. Runs outside VPC. @@ -67,6 +67,29 @@ Two separate CashPro APIs are used, each with its own OAuth credentials: - Production: `https://api.bofa.com` - Sandbox: `https://api-sb.bofa.com` +## Bank reconciliation (fetchBoaTransactions) + +The scheduled Lambda reconciles the Previous Day feed onto `payment#` records (pure logic lives in `src/boaRecon.js`, covered by `npm test`). + +**Classification.** Transactions are classified via a BAI transaction-code map (`BAI_CODE_EVENTS`; only 475 = check paid debit is empirically confirmed so far — the remaining codes are pending an enumeration replay over known event dates) with a fallback classifier over the statement description formats (ARP refer-to-maker, return-of-posted-check check-numbered and electronic variants, ACH CCD `DES:PAYMENTS ID:PMT` lines). Unmapped codes on check-shaped transactions are logged (`console.error`) and counted in the run summary — never silently dropped. + +**Matching.** Check events match on check number AND amount, evaluating all candidates (bank postings can drop/collapse digits on long check numbers). A number match with the wrong amount — or an unknown number — falls back to an exact-amount match within checks issued in the last 120 days; zero or multiple fallback candidates means unmatched, and unmatched transactions are recorded in the run summary with no write. Electronic returns (no check number) match by exact amount among bank-confirmed payments. + +**State transitions.** Every write sets BOTH `status` and `clear_status` (`clear_status` is bank truth; "Cleared without a subsequent return is permanent" keys off it): + +| Bank event | Result | +|---|---| +| Paid debit | `status=Cleared`, `clear_status=Cleared`, `paid_date`, `cleared_date`, `bank_reference` | +| Return credit (even if currently Cleared) | `clear_status=Returned`, `returned_date`; `status` re-written unchanged (the CSV ladder has no Returned rung) | +| Second paid debit on a Returned check | Redeposit: back to Cleared with new dates | +| Return on a Stampli-voided check | Terminal voided-and-bounced (`clear_status=Returned`, cancel status preserved), counted separately | + +Each applied event is appended to a `history` list attribute (`{event, date, bankRef, amount}`); identical replayed events are idempotent noops. + +**Replay.** The handler accepts an optional payload `{"fromDate": "YYYY-MM-DD", "toDate": "YYYY-MM-DD"}` (strictly validated; `toDate` defaults to `fromDate`; no payload = yesterday) for weekend/outage gap replays and BAI-code enumeration runs. + +**Run summary.** Each run writes a `boa_recon#` item (90-day TTL) with counts per classified event type, matched/applied/redeposit/voided-and-bounced totals, the unmatched check numbers and amounts, and any unknown BAI codes encountered. Unmatched and unknown-code transactions also `console.error` (Slack alerting is tracked in #71). + ## Documentation The canonical map of Sea Haven's AWS infrastructure lives in Confluence. This project's `payments-dashboard` stack is represented there as a Mermaid subgraph. diff --git a/package.json b/package.json index aadcb98..5f7be0f 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,9 @@ "files": [ "src/" ], + "scripts": { + "test": "node --test \"tests/**/*.test.js\"" + }, "dependencies": { "@aws-sdk/client-dynamodb": "^3.1087.0", "@aws-sdk/client-lambda": "^3.1087.0", diff --git a/src/boaRecon.js b/src/boaRecon.js new file mode 100644 index 0000000..6a8c4bb --- /dev/null +++ b/src/boaRecon.js @@ -0,0 +1,327 @@ +// Pure reconciliation logic for fetchBoaTransactions (payments-dashboard#66). +// No AWS clients or environment access here so node:test can exercise the +// classifier, matcher, and state transitions directly. + +import { toISODate } from "./dates.js"; + +// Untrusted values (bank descriptions, references) must be JSON-encoded +// before log interpolation — bank text can carry newlines, which would forge +// CloudWatch log lines (CWE-117). Same rule as processPaymentCsv's logSafe. +export const logSafe = (v) => JSON.stringify(String(v ?? "").slice(0, 128)); + +// Comma-tolerant amount parsing ("1,234.56" bank strings and stored values). +export const parseAmount = (value) => { + const num = parseFloat(String(value ?? "0").replace(/,/g, "").trim()); + return isNaN(num) ? 0 : num; +}; + +const cents = (v) => Math.round(Math.abs(parseAmount(v)) * 100); + +// Sign-insensitive cent equality; zero never matches (a blank amount must +// not pair with another blank amount). +export const amountsEqual = (a, b) => cents(a) > 0 && cents(a) === cents(b); + +// BAI transaction-code -> event map. BoA's Previous Day feed uses standard +// BAI type codes; only 475 (check paid, debit) is empirically confirmed so +// far. The codes for ARP refer-to-maker return credits, return-of-posted- +// check credits (check-numbered and electronic variants), ACH CCD debits and +// return credits, and the true meaning of 255 (the old filter assumed +// "returned check", unverified) are pending the enumeration replay over +// known event dates (2/20, 3/31, 4/8, 6/16-6/23) — add them here as they +// are confirmed. Until then classification falls back to description text, +// and unmapped codes on check-shaped transactions are logged and counted, +// never silently dropped. +export const BAI_CODE_EVENTS = { + 475: { event: "check_paid", direction: "debit" }, +}; + +// Standard BAI ranges: 100-399 are credit type codes, 400-699 are debit +// type codes. Used only when the feed carries no explicit indicator. +export function directionFromCode(code) { + const n = parseInt(String(code ?? ""), 10); + if (!Number.isInteger(n)) return null; + if (n >= 100 && n < 400) return "credit"; + if (n >= 400 && n < 700) return "debit"; + return null; +} + +export function directionOf(txn) { + const indicator = String( + txn.debitCreditIndicator ?? txn.creditDebitIndicator ?? "" + ).toUpperCase(); + if (indicator.includes("DEBIT")) return "debit"; + if (indicator.includes("CREDIT")) return "credit"; + const fromCode = directionFromCode(txn.transactionCode); + if (fromCode) return fromCode; + const amt = parseAmount(txn.amount); + if (amt < 0) return "debit"; + if (amt > 0) return "credit"; + return null; +} + +// Statement description formats observed on the 2026-07-21 reconciliation. +const ARP_RETURN_RE = /^ARP RETURNED CHECK REFER TO MAKER CHECK #\s*(\d+)\b/i; +const POSTED_RETURN_CHECK_RE = + /^RETURN OF POSTED CHECK \/ ITEM \(RECEIVED ON \d{2}-\d{2}\)\s*CHECK #\s*(\d+)\b/i; +const POSTED_RETURN_ELECTRONIC_RE = + /^RETURN OF POSTED CHECK \/ ITEM \(RECEIVED ON \d{2}-\d{2}\)\s*ELECTRONIC TRANSACTION\b/i; +const ACH_PMT_RE = /DES:PAYMENTS\s+ID:PMT\s*(\d+)/i; +const PMT_INFO_RE = /PMT INFO:\s*(.*)$/i; +const CHECK_PAID_RE = /^CHECK\s*#?\s*(\d+)$/i; +// Trailing digit run at the end of PMT INFO — Stampli recently started +// embedding the payment number there, and the bank wraps it with arbitrary +// internal spaces ("21 222000108", "2122200 0256"). +const EMBEDDED_NUMBER_RE = /(\d[\d ]{6,}\d)\s*$/; + +const stripLeadingZeros = (s) => String(s ?? "").replace(/^0+(?=\d)/, ""); + +// Classify one Previous Day transaction into a reconciliation event. +// Precedence: confirmed BAI code map first, then description text. Anything +// unmapped that still looks check/payment-shaped comes back as "unknown" so +// the caller can log and count it — never silently drop it. Everything else +// (transfers, misc bank activity) is "ignored". +export function classifyTransaction(txn) { + const code = String(txn.transactionCode ?? "").trim(); + const description = String( + txn.text ?? txn.description ?? txn.transactionDescription ?? "" + ).trim(); + const customerReference = stripLeadingZeros(String(txn.customerReference ?? "").trim()); + const bankReference = String(txn.bankReference ?? "").trim(); + const amount = Math.abs(parseAmount(txn.amount)); + const direction = BAI_CODE_EVENTS[code]?.direction ?? directionOf(txn); + + const base = { + code, + description, + customerReference, + bankReference, + amount, + direction, + checkNumber: null, + pmtId: null, + embeddedPaymentNumber: null, + vendorText: null, + checkShaped: false, + }; + + // Description facts, extracted regardless of code so a code-mapped event + // still carries the check number / PMT id it references. + let descriptionEvent = null; + let m; + if ((m = ARP_RETURN_RE.exec(description))) { + descriptionEvent = "check_return"; + base.checkNumber = stripLeadingZeros(m[1]); + } else if ((m = POSTED_RETURN_CHECK_RE.exec(description))) { + descriptionEvent = "check_return"; + base.checkNumber = stripLeadingZeros(m[1]); + } else if (POSTED_RETURN_ELECTRONIC_RE.test(description)) { + descriptionEvent = "electronic_return"; + } else if ((m = ACH_PMT_RE.exec(description))) { + base.pmtId = m[1]; + const info = PMT_INFO_RE.exec(description); + if (info) { + let vendorText = info[1].trim(); + const embedded = EMBEDDED_NUMBER_RE.exec(vendorText); + if (embedded && embedded[1].replace(/ /g, "").length >= 8) { + base.embeddedPaymentNumber = embedded[1].replace(/ /g, ""); + vendorText = vendorText.slice(0, embedded.index).trim(); + } + base.vendorText = vendorText || null; + } + if (direction === "debit") descriptionEvent = "ach_debit"; + else if (direction === "credit") descriptionEvent = "ach_return"; + // direction unknown -> leave null; falls through to "unknown" below. + } else if ((m = CHECK_PAID_RE.exec(description))) { + descriptionEvent = "check_paid"; + base.checkNumber = stripLeadingZeros(m[1]); + } + + const event = BAI_CODE_EVENTS[code]?.event ?? descriptionEvent; + + if (event === "check_paid" && !base.checkNumber) { + base.checkNumber = customerReference || null; + } + if (event === "check_return" && !base.checkNumber) { + base.checkNumber = customerReference || null; + } + + if (event) return { ...base, event }; + + base.checkShaped = + Boolean(customerReference) || /CHECK/i.test(description) || Boolean(base.pmtId); + return { ...base, event: base.checkShaped ? "unknown" : "ignored" }; +} + +// Days from a stored Stampli send date (canonical MM/DD/YYYY) to an ISO +// reference date; null when the stored date does not parse. +function daysSinceIssue(payment, refISO) { + const iso = toISODate(payment.send_payment_on); + if (!iso) return null; + return Math.round((Date.parse(refISO) - Date.parse(iso)) / 86400000); +} + +const issuedWithinDays = (payment, refISO, days) => { + const d = daysSinceIssue(payment, refISO); + return d !== null && d >= 0 && d <= days; +}; + +// Match a check event (paid debit or return credit) against payment records. +// Match on check number AND amount, evaluating all candidates — never +// first-match on number alone: bank postings drop/collapse digits on long +// check numbers, so a posting under one number can belong to another check +// (or to a number we never issued). When the number matches but the amount +// does not — or the number is unknown — fall back to an exact-amount match +// within checks issued in the last 120 days. Anything but a single fallback +// candidate is unmatched: no write on ambiguity. +export function matchCheckTransaction(classified, payments, refDateISO) { + const { checkNumber, amount } = classified; + const checks = payments.filter((p) => p.method === "Check" && p.check_number); + + const numberMatches = checkNumber + ? checks.filter((p) => p.check_number === checkNumber) + : []; + const exact = numberMatches.filter((p) => amountsEqual(p.amount_usd, amount)); + if (exact.length === 1) return { payment: exact[0], matchedBy: "number+amount" }; + if (exact.length > 1) return { unmatched: "multiple number+amount matches" }; + + const fallback = checks.filter( + (p) => amountsEqual(p.amount_usd, amount) && issuedWithinDays(p, refDateISO, 120) + ); + if (fallback.length === 1) return { payment: fallback[0], matchedBy: "amount" }; + if (fallback.length > 1) { + return { unmatched: `amount fallback ambiguous (${fallback.length} candidates)` }; + } + return { + unmatched: numberMatches.length + ? "number matched with wrong amount; no unique amount fallback" + : "no number match; no unique amount fallback", + }; +} + +// Electronic return credits carry no check number at all. Match by exact +// amount among bank-confirmed payments issued in the last 120 days +// (clear_status Cleared, or Returned so replays of an already-applied +// return dedupe to a noop instead of alerting). Ambiguity is unmatched. +export function matchElectronicReturn(classified, payments, refDateISO) { + const candidates = payments.filter( + (p) => + (p.clear_status === "Cleared" || p.clear_status === "Returned") && + amountsEqual(p.amount_usd, classified.amount) && + issuedWithinDays(p, refDateISO, 120) + ); + if (candidates.length === 1) return { payment: candidates[0], matchedBy: "amount+cleared" }; + if (candidates.length > 1) { + return { unmatched: `electronic return ambiguous (${candidates.length} candidates)` }; + } + return { unmatched: "no bank-confirmed payment with this amount" }; +} + +const CANCEL_STATUSES = ["voided", "cancelled", "canceled", "marked as void"]; +export const isCancelStatus = (s) => CANCEL_STATUSES.includes(String(s ?? "").toLowerCase()); + +// Decide the DDB write for a matched bank event. Pure: returns the fields +// to set plus the history entry; the handler turns it into an UpdateCommand +// and mirrors the fields onto its in-memory copy. +// +// Invariant: every non-noop result sets BOTH status and clear_status — the +// 2026-07-21 reconciliation traced five missed returns ($5,256.62) to +// writers touching one field but not the other. +// +// Status semantics (documented decision, #66): `clear_status` is bank truth +// ("Cleared without a subsequent return is permanent" keys off it). `status` +// stays on the CSV lifecycle ladder, which has no "Returned" rung, so on a +// return it is re-written with its current value to keep the both-fields +// invariant; consumers that need bounce visibility read clear_status. On a +// paid debit against a canceled record (a voided check the bank paid — +// expected without Positive Pay), the cancel status is likewise preserved: +// the ARP return that follows lands as terminal voided-and-bounced. +export function applyEvent(payment, classified, eventDateISO) { + const { event, amount, bankReference } = classified; + const canceled = isCancelStatus(payment.status); + const historyEvent = { + event, + date: eventDateISO, + bankRef: bankReference, + amount, + }; + + // Replay idempotence: an identical event already recorded in history is a + // noop, so re-running a date range never double-applies transitions. + const alreadyApplied = (payment.history || []).some( + (h) => + h.event === historyEvent.event && + h.date === historyEvent.date && + h.bankRef === historyEvent.bankRef && + amountsEqual(h.amount, amount) + ); + if (alreadyApplied) return { kind: "noop", updates: null, historyEvent: null }; + + if (event === "check_paid" || event === "ach_debit") { + const redeposit = payment.clear_status === "Returned"; + const updates = { + status: canceled ? payment.status : "Cleared", + clear_status: "Cleared", + paid_date: eventDateISO, + cleared_date: eventDateISO, + bank_reference: bankReference, + }; + if (event === "ach_debit" && classified.pmtId) updates.pmt_id = classified.pmtId; + return { + kind: redeposit ? "redeposit" : canceled ? "cleared_on_canceled" : "cleared", + updates, + historyEvent, + }; + } + + if (event === "check_return" || event === "electronic_return" || event === "ach_return") { + const updates = { + status: payment.status ?? "", + clear_status: "Returned", + returned_date: eventDateISO, + bank_reference: bankReference, + }; + if (event === "ach_return" && classified.pmtId) updates.pmt_id = classified.pmtId; + return { + // A return against a canceled record is the expected void-then-bounce + // ARP cycle: terminal voided-and-bounced, counted separately. + kind: canceled ? "voided_and_bounced" : "returned", + updates, + historyEvent, + }; + } + + return { kind: "noop", updates: null, historyEvent: null }; +} + +const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/; + +export function isValidISODate(s) { + if (typeof s !== "string" || !ISO_DATE_RE.test(s)) return false; + const [y, mo, d] = s.split("-").map(Number); + const dt = new Date(Date.UTC(y, mo - 1, d)); + return ( + dt.getUTCFullYear() === y && dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d + ); +} + +// Optional {fromDate, toDate} replay payload; the default stays yesterday, +// matching the previous-day inquiry cadence. Strings are validated strictly +// so a malformed payload fails loudly instead of querying a garbage range. +export function resolveDateRange(event, now = new Date()) { + const hasFrom = event?.fromDate != null; + const hasTo = event?.toDate != null; + if (!hasFrom && !hasTo) { + const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000); + const d = yesterday.toISOString().split("T")[0]; + return { fromDate: d, toDate: d }; + } + const fromDate = hasFrom ? event.fromDate : event.toDate; + const toDate = hasTo ? event.toDate : event.fromDate; + if (!isValidISODate(fromDate) || !isValidISODate(toDate)) { + throw new Error("fromDate/toDate must be valid YYYY-MM-DD strings"); + } + if (fromDate > toDate) { + throw new Error("fromDate must be <= toDate"); + } + return { fromDate, toDate }; +} diff --git a/src/fetchBoaTransactions.js b/src/fetchBoaTransactions.js index 660e8ab..a877e1f 100644 --- a/src/fetchBoaTransactions.js +++ b/src/fetchBoaTransactions.js @@ -1,6 +1,15 @@ import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; -import { DynamoDBDocumentClient, ScanCommand, UpdateCommand } from "@aws-sdk/lib-dynamodb"; +import { DynamoDBDocumentClient, PutCommand, ScanCommand, UpdateCommand } from "@aws-sdk/lib-dynamodb"; import { SecretsManagerClient, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager"; +import { + applyEvent, + classifyTransaction, + isValidISODate, + logSafe, + matchCheckTransaction, + matchElectronicReturn, + resolveDateRange, +} from "./boaRecon.js"; const ddb = DynamoDBDocumentClient.from(new DynamoDBClient()); const secrets = new SecretsManagerClient(); @@ -28,24 +37,21 @@ async function getAccessToken(applicationID, clientId, clientSecret) { }); if (!res.ok) { - const text = await res.text(); - throw new Error(`OAuth token exchange failed: ${res.status} - ${text}`); + throw new Error(`OAuth token exchange failed: HTTP ${res.status}`); } const data = await res.json(); return data.access_token; } -export const handler = async () => { +export const handler = async (event) => { + // Optional replay payload {fromDate, toDate}; default is yesterday. + const { fromDate, toDate } = resolveDateRange(event ?? {}); + const { appId, clientId, token: clientSecret, accountNumber, bankId } = await getReportingCreds(); const bearerToken = await getAccessToken(appId, clientId, clientSecret); - // Get yesterday's date in YYYY-MM-DD - const yesterday = new Date(); - yesterday.setDate(yesterday.getDate() - 1); - const dateStr = yesterday.toISOString().split("T")[0]; - // Call CashPro Previous Day Transaction Inquiry const res = await fetch(`${BOA_BASE_URL}/cashpro/reporting/v1/transaction-inquiries/previous-day`, { method: "POST", @@ -54,15 +60,14 @@ export const handler = async () => { Authorization: `Bearer ${bearerToken}`, }, body: JSON.stringify({ - fromDate: dateStr, - toDate: dateStr, + fromDate, + toDate, accounts: [{ accountNumber, bankId }], }), }); if (!res.ok) { - const text = await res.text(); - throw new Error(`BoA API error ${res.status}: ${text}`); + throw new Error(`BoA API error: HTTP ${res.status}`); } const data = await res.json(); @@ -72,17 +77,14 @@ export const handler = async () => { (acct) => acct.transactions || [] ); - // Filter for cleared checks (475) and returned checks (255) - const relevant = allTransactions.filter( - (t) => t.transactionCode === "475" || t.transactionCode === "255" + // Process in posting-date order so multi-day replays apply paid -> return + // -> redeposit sequences in the order the bank did. + allTransactions.sort((a, b) => + String(a.valueDate ?? "").localeCompare(String(b.valueDate ?? "")) ); - if (!relevant.length) { - console.log(`No check transactions (475/255) found for ${dateStr}`); - return { statusCode: 200, body: `No relevant transactions for ${dateStr}` }; - } - - // Load all check payments from DynamoDB to match against + // Load check payments from DynamoDB to match against. (#69 extends this + // to ACH by dropping the method filter.) const payments = []; let lastKey; do { @@ -99,44 +101,147 @@ export const handler = async () => { lastKey = result.LastEvaluatedKey; } while (lastKey); - let matched = 0; + const summary = { + transactions_seen: allTransactions.length, + classified: {}, + matched: 0, + applied: 0, + already_applied: 0, + redeposits: 0, + voided_and_bounced: 0, + unmatched: [], + unknown_codes: {}, + }; - for (const txn of relevant) { - const custRef = (txn.customerReference || "").replace(/^0+/, ""); - const bankRef = txn.bankReference || ""; + for (const txn of allTransactions) { + const classified = classifyTransaction(txn); + summary.classified[classified.event] = (summary.classified[classified.event] || 0) + 1; - // Match customerReference (check number with leading zeros stripped) to our check_number - const matchedPayment = payments.find((p) => - p.check_number && p.check_number === custRef - ); + if (classified.event === "ignored") continue; - if (!matchedPayment) { - console.log(`No match for customerReference: ${txn.customerReference} (bankRef: ${bankRef})`); + if (classified.event === "unknown") { + summary.unknown_codes[classified.code || "?"] = + (summary.unknown_codes[classified.code || "?"] || 0) + 1; + console.error( + `Unknown check-shaped transaction: code=${logSafe(classified.code)}, ` + + `description=${logSafe(classified.description)}, ` + + `ref=${logSafe(classified.customerReference)}, amount=${classified.amount}` + ); continue; } - const clearStatus = txn.transactionCode === "475" ? "Cleared" : "Returned"; + if (classified.event === "ach_debit" || classified.event === "ach_return") { + // ACH bank confirmation lands with #69. + console.log(`ACH ${classified.event} classified (PMT ${logSafe(classified.pmtId)}); deferred to #69`); + continue; + } + + const eventDate = isValidISODate(txn.valueDate) ? txn.valueDate : toDate; + + const match = + classified.event === "electronic_return" + ? matchElectronicReturn(classified, payments, eventDate) + : matchCheckTransaction(classified, payments, eventDate); + + if (!match.payment) { + summary.unmatched.push({ + event: classified.event, + check_number: classified.checkNumber, + amount: classified.amount, + reason: match.unmatched, + }); + console.error( + `Unmatched ${classified.event}: check=${logSafe(classified.checkNumber)}, ` + + `amount=${classified.amount}, reason=${match.unmatched} ` + + `(bankRef: ${logSafe(classified.bankReference)})` + ); + continue; + } + + summary.matched++; + const applied = applyEvent(match.payment, classified, eventDate); + if (applied.kind === "noop") { + summary.already_applied++; + continue; + } + if (applied.kind === "redeposit") summary.redeposits++; + if (applied.kind === "voided_and_bounced") summary.voided_and_bounced++; + + const sets = ["#history = list_append(if_not_exists(#history, :empty), :hist)"]; + const names = { "#history": "history" }; + const values = { ":empty": [], ":hist": [applied.historyEvent] }; + Object.entries(applied.updates).forEach(([field, value], i) => { + names[`#f${i}`] = field; + values[`:v${i}`] = value; + sets.push(`#f${i} = :v${i}`); + }); await ddb.send( new UpdateCommand({ TableName: TABLE_NAME, - Key: { pk: matchedPayment.pk }, - UpdateExpression: "SET clear_status = :status, bank_reference = :ref, cleared_date = :date", - ExpressionAttributeValues: { - ":status": clearStatus, - ":ref": bankRef, - ":date": txn.valueDate || dateStr, - }, + Key: { pk: match.payment.pk }, + UpdateExpression: `SET ${sets.join(", ")}`, + ExpressionAttributeNames: names, + ExpressionAttributeValues: values, }) ); + summary.applied++; - matched++; - console.log(`${clearStatus}: check ${matchedPayment.check_number} (bankRef: ${bankRef})`); + // Mirror the write onto the in-memory copy so later transactions in the + // same run (return after paid, redeposit after return) see current state. + Object.assign(match.payment, applied.updates); + match.payment.history = [...(match.payment.history || []), applied.historyEvent]; + + console.log( + `${applied.kind}: check ${logSafe(match.payment.check_number)} via ${match.matchedBy} ` + + `(bankRef: ${logSafe(classified.bankReference)})` + ); } - console.log(`Processed ${relevant.length} transactions, matched ${matched} payments`); + // Run summary record for auditing/alerting (Slack wiring is #71). + const runDate = fromDate === toDate ? fromDate : `${fromDate}_${toDate}`; + await ddb.send( + new PutCommand({ + TableName: TABLE_NAME, + Item: { + pk: `boa_recon#${runDate}`, + run_at: new Date().toISOString(), + from_date: fromDate, + to_date: toDate, + transactions_seen: summary.transactions_seen, + classified: summary.classified, + matched: summary.matched, + applied: summary.applied, + already_applied: summary.already_applied, + redeposits: summary.redeposits, + voided_and_bounced: summary.voided_and_bounced, + unmatched_count: summary.unmatched.length, + unmatched: summary.unmatched, + unknown_codes: summary.unknown_codes, + ttl: Math.floor(Date.now() / 1000) + 90 * 24 * 60 * 60, + }, + }) + ); + + const unknownCount = Object.values(summary.unknown_codes).reduce((a, b) => a + b, 0); + if (summary.unmatched.length || unknownCount) { + console.error( + `Reconciliation ${fromDate}..${toDate}: ${summary.unmatched.length} unmatched, ` + + `${unknownCount} unknown-code transactions (see boa_recon#${runDate})` + ); + } + + console.log( + `Processed ${summary.transactions_seen} transactions ${fromDate}..${toDate}: ` + + `${summary.matched} matched, ${summary.applied} applied, ` + + `${summary.already_applied} already applied, ${summary.redeposits} redeposits, ` + + `${summary.voided_and_bounced} voided-and-bounced, ${summary.unmatched.length} unmatched` + ); + return { statusCode: 200, - body: `${matched} of ${relevant.length} transactions matched to payments`, + body: + `${summary.matched} of ${summary.transactions_seen} transactions matched ` + + `(${summary.applied} applied, ${summary.unmatched.length} unmatched)`, }; }; diff --git a/tests/boaRecon.test.js b/tests/boaRecon.test.js new file mode 100644 index 0000000..dbc25e2 --- /dev/null +++ b/tests/boaRecon.test.js @@ -0,0 +1,394 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + BAI_CODE_EVENTS, + amountsEqual, + applyEvent, + classifyTransaction, + isCancelStatus, + isValidISODate, + matchCheckTransaction, + matchElectronicReturn, + resolveDateRange, +} from "../src/boaRecon.js"; + +// Description fixtures are real statement lines from the 2026-07-21 +// reconciliation (amounts/refs anonymized where they don't matter). + +test("classifier: BAI code 475 is a check paid debit", () => { + const c = classifyTransaction({ + transactionCode: "475", + customerReference: "0001122200030", + bankReference: "813312345", + amount: "1,892.62", + }); + assert.equal(c.event, "check_paid"); + assert.equal(c.direction, "debit"); + assert.equal(c.checkNumber, "1122200030"); + assert.equal(c.amount, 1892.62); + assert.equal(BAI_CODE_EVENTS["475"].event, "check_paid"); +}); + +test("classifier: statement-style 'Check ' description is check paid", () => { + const c = classifyTransaction({ text: "Check 3176", amount: "-350" }); + assert.equal(c.event, "check_paid"); + assert.equal(c.checkNumber, "3176"); +}); + +test("classifier: ARP refer-to-maker return credit", () => { + const c = classifyTransaction({ + transactionCode: "354", + text: "ARP RETURNED CHECK REFER TO MAKER CHECK # 1122200030 PAID DATE 02/19/26", + amount: "1,892.62", + }); + assert.equal(c.event, "check_return"); + assert.equal(c.direction, "credit"); + assert.equal(c.checkNumber, "1122200030"); +}); + +test("classifier: return of posted check, check-numbered variant", () => { + const c = classifyTransaction({ + text: "RETURN OF POSTED CHECK / ITEM (RECEIVED ON 03-02) CHECK #1122200236", + amount: "3,350.00", + }); + assert.equal(c.event, "check_return"); + assert.equal(c.checkNumber, "1122200236"); +}); + +test("classifier: return of posted check, electronic variant has no check number", () => { + const c = classifyTransaction({ + text: "RETURN OF POSTED CHECK / ITEM (RECEIVED ON 03-02) ELECTRONIC TRANSACTION", + amount: "3,992.15", + }); + assert.equal(c.event, "electronic_return"); + assert.equal(c.checkNumber, null); +}); + +test("classifier: ACH CCD debit without embedded payment number", () => { + const c = classifyTransaction({ + text: "FISK EXCAVATING DES:PAYMENTS ID:PMT 6814295 INDN:Sea Haven Industries, CO ID:1811679038 CCD PMT INFO:Fisk Excavating Inc", + amount: "-650", + }); + assert.equal(c.event, "ach_debit"); + assert.equal(c.pmtId, "6814295"); + assert.equal(c.embeddedPaymentNumber, null); + assert.equal(c.vendorText, "Fisk Excavating Inc"); +}); + +test("classifier: ACH debit with embedded payment number", () => { + const c = classifyTransaction({ + text: "COBRA SEPTIC DES:PAYMENTS ID:PMT 7584198 INDN:Sea Haven Industries, CO ID:1811679038 CCD PMT INFO:Cobra Septic 21222000264", + amount: "-8,300.00", + }); + assert.equal(c.event, "ach_debit"); + assert.equal(c.pmtId, "7584198"); + assert.equal(c.embeddedPaymentNumber, "21222000264"); + assert.equal(c.vendorText, "Cobra Septic"); +}); + +test("classifier: embedded payment number with internal space '21 222000273'", () => { + const c = classifyTransaction({ + text: "HALL PUMP SALES DES:PAYMENTS ID:PMT 7617407 INDN:Sea Haven Industries, CO ID:1811679038 CCD PMT INFO:Hall Pump Sales & Service Corporation 21 222000273", + amount: "-5,495.00", + }); + assert.equal(c.embeddedPaymentNumber, "21222000273"); + assert.equal(c.vendorText, "Hall Pump Sales & Service Corporation"); +}); + +test("classifier: embedded payment number with internal space '2122200 0256'", () => { + const c = classifyTransaction({ + text: "STERLING SEPTIC DES:PAYMENTS ID:PMT 7603631 INDN:Sea Haven Industries, CO ID:1811679038 CCD PMT INFO:Sterling Septic & Plumbing, LLC. 2122200 0256", + amount: "-1,407.59", + }); + assert.equal(c.embeddedPaymentNumber, "21222000256"); + assert.equal(c.vendorText, "Sterling Septic & Plumbing, LLC."); +}); + +test("classifier: ACH reversal credit reusing PMT id is ach_return", () => { + const c = classifyTransaction({ + text: "SEA HAVEN INDUST DES:PAYMENTS ID:PMT 7176466 INDN:Sea Haven Industries, CO ID:1811679038 CCD PMT INFO:Sea Haven Industries, Inc 21222000128", + amount: "2,100.00", + }); + assert.equal(c.event, "ach_return"); + assert.equal(c.pmtId, "7176466"); + assert.equal(c.embeddedPaymentNumber, "21222000128"); +}); + +test("classifier: short trailing digits are not an embedded payment number", () => { + const c = classifyTransaction({ + text: "SOME VENDOR DES:PAYMENTS ID:PMT 700001 INDN:Sea Haven Industries, CO ID:1811679038 CCD PMT INFO:Vendor Company 2000", + amount: "-100.00", + }); + assert.equal(c.embeddedPaymentNumber, null); + assert.equal(c.vendorText, "Vendor Company 2000"); +}); + +test("classifier: unknown code on a check-shaped transaction is 'unknown', never dropped", () => { + const c = classifyTransaction({ + transactionCode: "699", + customerReference: "3301", + text: "SOMETHING NEW", + amount: "-100.00", + }); + assert.equal(c.event, "unknown"); + assert.equal(c.checkShaped, true); + assert.equal(c.code, "699"); +}); + +test("classifier: transfers and misc bank activity are ignored", () => { + const c = classifyTransaction({ + text: "ACCOUNT TRANSFER TRSF FROM 483096772516", + amount: "15,000.00", + }); + assert.equal(c.event, "ignored"); + // Not check-shaped (no code, no reference, no CHECK text) -> ignored too. + const c2 = classifyTransaction({ text: "RETURN ITEM CHARGEBACK", amount: "-215" }); + assert.equal(c2.event, "ignored"); +}); + +test("amountsEqual: comma-tolerant, sign-insensitive, zero never matches", () => { + assert.equal(amountsEqual("1,892.62", -1892.62), true); + assert.equal(amountsEqual(0, 0), false); + assert.equal(amountsEqual("", ""), false); +}); + +// ---------------------------------------------------------------- matcher + +const payment = (over = {}) => ({ + pk: `payment#${over.check_number ?? "1001"}`, + check_number: "1001", + method: "Check", + amount_usd: 500, + status: "Outstanding", + send_payment_on: "07/01/2026", + ...over, +}); + +test("matcher: check number AND amount must both match", () => { + const payments = [payment({ check_number: "1001", amount_usd: 500 })]; + const m = matchCheckTransaction({ checkNumber: "1001", amount: 500 }, payments, "2026-07-20"); + assert.equal(m.payment, payments[0]); + assert.equal(m.matchedBy, "number+amount"); +}); + +test("matcher: number match with wrong amount falls back to unique exact-amount match", () => { + const payments = [ + payment({ check_number: "1001", amount_usd: 500 }), + payment({ check_number: "1002", amount_usd: 750.25, pk: "payment#1002" }), + ]; + // Posting collapsed onto 1001's number but carries 1002's amount. + const m = matchCheckTransaction({ checkNumber: "1001", amount: 750.25 }, payments, "2026-07-20"); + assert.equal(m.payment.check_number, "1002"); + assert.equal(m.matchedBy, "amount"); +}); + +test("matcher: collapsed number we never issued still recovers by amount", () => { + const payments = [payment({ check_number: "11222000123", amount_usd: 6413 })]; + const m = matchCheckTransaction({ checkNumber: "1222000012", amount: 6413 }, payments, "2026-07-20"); + assert.equal(m.payment.check_number, "11222000123"); + assert.equal(m.matchedBy, "amount"); +}); + +test("matcher: ambiguous amount fallback is unmatched — no write", () => { + const payments = [ + payment({ check_number: "1001", amount_usd: 500 }), + payment({ check_number: "1002", amount_usd: 350, pk: "payment#1002" }), + payment({ check_number: "1003", amount_usd: 350, pk: "payment#1003" }), + ]; + const m = matchCheckTransaction({ checkNumber: "1001", amount: 350 }, payments, "2026-07-20"); + assert.equal(m.payment, undefined); + assert.match(m.unmatched, /ambiguous/); +}); + +test("matcher: zero candidates is unmatched", () => { + const m = matchCheckTransaction({ checkNumber: "9999", amount: 123.45 }, [payment()], "2026-07-20"); + assert.equal(m.payment, undefined); +}); + +test("matcher: amount fallback only considers checks issued in the last 120 days", () => { + const payments = [ + payment({ check_number: "1001", amount_usd: 350, send_payment_on: "01/02/2026" }), + ]; + const m = matchCheckTransaction({ checkNumber: "2001", amount: 350 }, payments, "2026-07-20"); + assert.equal(m.payment, undefined); +}); + +test("matcher: non-Check payments are never check-match candidates", () => { + const payments = [payment({ method: "ACH", check_number: "21222000264", amount_usd: 350 })]; + const m = matchCheckTransaction({ checkNumber: "21222000264", amount: 350 }, payments, "2026-07-20"); + assert.equal(m.payment, undefined); +}); + +test("electronic return: unique amount among bank-confirmed payments matches", () => { + const payments = [ + payment({ check_number: "1001", amount_usd: 3992.15, clear_status: "Cleared" }), + payment({ check_number: "1002", amount_usd: 3992.15, pk: "payment#1002" }), // not bank-confirmed + ]; + const m = matchElectronicReturn({ amount: 3992.15 }, payments, "2026-07-20"); + assert.equal(m.payment.check_number, "1001"); +}); + +test("electronic return: ambiguity is unmatched", () => { + const payments = [ + payment({ check_number: "1001", amount_usd: 2500, clear_status: "Cleared" }), + payment({ check_number: "1002", amount_usd: 2500, clear_status: "Cleared", pk: "payment#1002" }), + ]; + const m = matchElectronicReturn({ amount: 2500 }, payments, "2026-07-20"); + assert.equal(m.payment, undefined); + assert.match(m.unmatched, /ambiguous/); +}); + +// ------------------------------------------------------------ transitions + +const paidEvent = (over = {}) => ({ + event: "check_paid", + amount: 500, + bankReference: "813300001", + ...over, +}); +const returnEvent = (over = {}) => ({ + event: "check_return", + amount: 500, + bankReference: "813300002", + ...over, +}); + +test("transitions: paid debit sets BOTH status and clear_status to Cleared with dates", () => { + const p = payment(); + const r = applyEvent(p, paidEvent(), "2026-07-19"); + assert.equal(r.kind, "cleared"); + assert.equal(r.updates.status, "Cleared"); + assert.equal(r.updates.clear_status, "Cleared"); + assert.equal(r.updates.paid_date, "2026-07-19"); + assert.equal(r.updates.cleared_date, "2026-07-19"); + assert.equal(r.updates.bank_reference, "813300001"); + assert.deepEqual(r.historyEvent, { + event: "check_paid", + date: "2026-07-19", + bankRef: "813300001", + amount: 500, + }); +}); + +test("transitions: return applies EVEN IF currently Cleared; clear_status is bank truth", () => { + const p = payment({ status: "Cleared", clear_status: "Cleared" }); + const r = applyEvent(p, returnEvent(), "2026-07-20"); + assert.equal(r.kind, "returned"); + assert.equal(r.updates.clear_status, "Returned"); + assert.equal(r.updates.returned_date, "2026-07-20"); + // status has no "Returned" rung on the CSV ladder — re-written unchanged. + assert.equal(r.updates.status, "Cleared"); +}); + +test("transitions: second paid debit on a Returned check is a redeposit back to Cleared", () => { + const p = payment({ status: "Cleared", clear_status: "Returned" }); + const r = applyEvent(p, paidEvent({ bankReference: "813300003" }), "2026-07-21"); + assert.equal(r.kind, "redeposit"); + assert.equal(r.updates.status, "Cleared"); + assert.equal(r.updates.clear_status, "Cleared"); + assert.equal(r.updates.paid_date, "2026-07-21"); +}); + +test("transitions: paid -> returned -> redeposit sequence accumulates history", () => { + const p = payment(); + p.history = p.history || []; + const seq = [ + [paidEvent(), "2026-07-18"], + [returnEvent(), "2026-07-19"], + [paidEvent({ bankReference: "813300004" }), "2026-07-20"], + ]; + const kinds = []; + for (const [ev, date] of seq) { + const r = applyEvent(p, ev, date); + kinds.push(r.kind); + Object.assign(p, r.updates); + p.history.push(r.historyEvent); + } + assert.deepEqual(kinds, ["cleared", "returned", "redeposit"]); + assert.equal(p.clear_status, "Cleared"); + assert.equal(p.history.length, 3); + assert.deepEqual( + p.history.map((h) => h.event), + ["check_paid", "check_return", "check_paid"] + ); +}); + +test("transitions: return on a canceled record is terminal voided-and-bounced", () => { + const p = payment({ status: "Voided", clear_status: "Cleared" }); + const r = applyEvent(p, returnEvent(), "2026-07-20"); + assert.equal(r.kind, "voided_and_bounced"); + assert.equal(r.updates.clear_status, "Returned"); + assert.equal(r.updates.status, "Voided"); // cancel status stays visible +}); + +test("transitions: paid debit on a canceled record keeps the cancel status", () => { + const p = payment({ status: "Marked as Void" }); + const r = applyEvent(p, paidEvent(), "2026-07-19"); + assert.equal(r.kind, "cleared_on_canceled"); + assert.equal(r.updates.status, "Marked as Void"); + assert.equal(r.updates.clear_status, "Cleared"); +}); + +test("transitions: identical replayed event is a noop (idempotent replays)", () => { + const p = payment({ + status: "Cleared", + clear_status: "Cleared", + history: [{ event: "check_paid", date: "2026-07-19", bankRef: "813300001", amount: 500 }], + }); + const r = applyEvent(p, paidEvent(), "2026-07-19"); + assert.equal(r.kind, "noop"); + assert.equal(r.updates, null); +}); + +test("transitions: every non-noop result sets both status and clear_status", () => { + const cases = [ + [payment(), paidEvent()], + [payment({ status: "Cleared", clear_status: "Cleared" }), returnEvent()], + [payment({ status: "Cleared", clear_status: "Returned" }), paidEvent()], + [payment({ status: "Voided" }), returnEvent()], + [payment({ clear_status: "Cleared" }), { event: "electronic_return", amount: 500, bankReference: "x" }], + ]; + for (const [p, ev] of cases) { + const r = applyEvent(p, ev, "2026-07-20"); + assert.notEqual(r.kind, "noop"); + assert.ok("status" in r.updates, `${r.kind} must set status`); + assert.ok("clear_status" in r.updates, `${r.kind} must set clear_status`); + } +}); + +test("isCancelStatus covers the Stampli cancel vocabulary", () => { + for (const s of ["Voided", "cancelled", "Canceled", "Marked as Void"]) { + assert.equal(isCancelStatus(s), true); + } + assert.equal(isCancelStatus("Cleared"), false); +}); + +// ------------------------------------------------------------- date range + +test("resolveDateRange: default is yesterday", () => { + const r = resolveDateRange({}, new Date("2026-07-21T13:00:00Z")); + assert.deepEqual(r, { fromDate: "2026-07-20", toDate: "2026-07-20" }); +}); + +test("resolveDateRange: explicit valid range passes through", () => { + const r = resolveDateRange({ fromDate: "2026-06-16", toDate: "2026-06-23" }); + assert.deepEqual(r, { fromDate: "2026-06-16", toDate: "2026-06-23" }); +}); + +test("resolveDateRange: fromDate only replays a single day", () => { + const r = resolveDateRange({ fromDate: "2026-06-08" }); + assert.deepEqual(r, { fromDate: "2026-06-08", toDate: "2026-06-08" }); +}); + +test("resolveDateRange: malformed and impossible dates throw", () => { + assert.throws(() => resolveDateRange({ fromDate: "06/08/2026" })); + assert.throws(() => resolveDateRange({ fromDate: "2026-02-30" })); + assert.throws(() => resolveDateRange({ fromDate: "2026-07-02", toDate: "2026-07-01" })); + assert.throws(() => resolveDateRange({ fromDate: "2026-07-01; DROP", toDate: "2026-07-02" })); +}); + +test("isValidISODate rejects non-strings and bad calendar dates", () => { + assert.equal(isValidISODate("2026-07-20"), true); + assert.equal(isValidISODate("2026-13-01"), false); + assert.equal(isValidISODate(20260720), false); +});