From 817e1f6a74b16bb8f6364b5a1972336ac9207646 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Fri, 29 May 2026 14:30:44 -0400 Subject: [PATCH] Add synthetic export fixture and run classification quality gate in CI --- tests/fixtures/sample_export.csv | 31 ++++++ tests/test_classify.py | 185 +++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 tests/fixtures/sample_export.csv diff --git a/tests/fixtures/sample_export.csv b/tests/fixtures/sample_export.csv new file mode 100644 index 0000000..25b7552 --- /dev/null +++ b/tests/fixtures/sample_export.csv @@ -0,0 +1,31 @@ +WO Number,WO Description,Equipment Code,Organization,Due Date,Department,WO Status,Hold Reason,Last Comment,Last Comment By,Last Comment Date,Contractor,Contractor Description +WO-1001,Repair HVAC unit at dock 3,HVAC-001,ABQ5,2026-05-15,SSP,H,REPORT,
3rd attempt process for schedule confirmation. Vendor please confirm schedule start date and proceed.
,J.Smith,2026-05-10,Acme HVAC,Acme HVAC Services +WO-1002,Replace lighting ballast,LIGHT-042,ACY9,2026-05-16,RME,H,SCHEDULING,
2nd attempt process for schedule confirmation. Please provide a confirmed date for this work order.
,T.Jones,2026-05-11,Bright Electric,Bright Electric LLC +WO-1003,Fix water leak in restroom,PLUMB-007,BOS1,2026-05-17,SSP,IP,,
1st escalation sent to vendor. Awaiting response from contractor regarding scheduling.
,M.Davis,2026-05-12,Metro Plumbing,Metro Plumbing Inc +WO-1004,Inspect fire suppression system,FIRE-015,ABQ5,2026-05-18,SSP,R,,
SIM ticket TT-12345678 opened for this work order. Tracking in Amazon SIM system.
,K.Wilson,2026-05-13,FireSafe Co,FireSafe Company +WO-1005,Service generator unit,GEN-003,ACY9,2026-05-19,RME,R,,
Site tech reported vendor was a no show for Tuesday. Rescheduling required.
,P.Brown,2026-05-14,Power Gen LLC,Power Generation LLC +WO-1006,Calibrate dock door sensors,DOCK-022,BOS1,2026-05-20,SSP,IP,,
WO schedule confirmed with vendor for 5/22. All parties notified.
,L.Garcia,2026-05-15,Dock Systems,Dock Systems Inc +WO-1007,Weekly PM on conveyor belt,CONV-011,ABQ5,2026-05-21,RME,IP,,
Weekly WO scheduled. Service reports required EOD Friday per standard cadence.
,R.Martinez,2026-05-16,Belt Tech,Belt Tech Services +WO-1008,Replace worn floor mats,FLOOR-009,ACY9,2026-05-22,BBM,H,REPORT,,A.Thompson,2026-05-17,Clean Facility,Clean Facility Services +WO-1009,Inspect sprinkler heads,FIRE-031,BOS1,2026-05-23,SSP,H,SCHEDULING,,B.Anderson,2026-05-18,FireSafe Co,FireSafe Company +WO-1010,Order replacement motor,MOTOR-005,ABQ5,2026-05-24,RME,H,VENDOR,,C.Jackson,2026-05-19,Motor Supply,Motor Supply Co +WO-1011,Cancel duplicate work order,WO-DUP-001,ACY9,2026-05-25,SSP,RCAN,,
WO cancelled, created in error. Duplicate of WO-1010.
,D.White,2026-05-20,N/A,N/A +WO-1012,Perform task and close,TASK-088,BOS1,2026-05-26,RME,H,REPORT,
Vendor arrived and performed task. All work completed satisfactorily.
,E.Harris,2026-05-21,General Contractors,General Contractors LLC +WO-1013,PM on roof HVAC,HVAC-099,ABQ5,2026-05-27,SSP,R,,
WO schedule confirmed with vendor for 5/29 start time 0800.
,F.Clark,2026-05-22,Acme HVAC,Acme HVAC Services +WO-1014,Service dock bumpers weekly,DOCK-044,ACY9,2026-05-28,SSP,IP,,
Weekly WO scheduled. Service reports are required weekly per maintenance plan.
,G.Lewis,2026-05-23,Dock Systems,Dock Systems Inc +WO-1015,Replace battery backup,UPS-002,BOS1,2026-05-29,RME,H,SCHEDULING,
2nd attempt process for schedule confirmation. Vendor has not responded to prior outreach.
,H.Robinson,2026-05-24,Power Gen LLC,Power Generation LLC +WO-1016,Inspect emergency exits,EXIT-007,ABQ5,2026-05-30,SSP,R,,
3rd attempt process for schedule confirmation. Escalating to vendor management for response.
,I.Walker,2026-05-25,Safety First,Safety First Inc +WO-1017,Fix broken dock seal,DOCK-055,ACY9,2026-05-31,SSP,IP,,
Vendor confirmed schedule for 6/2 arrival.
,J.Hall,2026-05-26,Dock Systems,Dock Systems Inc +WO-1018,Service fire extinguishers,FIRE-022,BOS1,2026-06-01,SSP,RCAN,,
WO cancelled. Work was already completed under WO-1004.
,K.Young,2026-05-27,FireSafe Co,FireSafe Company +WO-1019,PM on cooling tower,COOL-001,ABQ5,2026-06-02,RME,H,REPORT,,L.Allen,2026-05-28,Cooling Tech,Cooling Tech LLC +WO-1020,Inspect electrical panel,ELEC-018,ACY9,2026-06-03,RME,H,VENDOR,,M.King,2026-05-29,Volt Electric,Volt Electric Services +WO-1021,Replace worn casters on cart,CART-003,BOS1,2026-06-04,BBM,IP,,
Procurement to provide PO for parts order before work can begin.
,N.Wright,2026-05-30,General Contractors,General Contractors LLC +WO-1022,Patch roof leak,ROOF-006,ABQ5,2026-06-05,SSP,IP,,
Copy.
,O.Scott,2026-05-31,Roof Masters,Roof Masters Inc +WO-1023,Service dock bumpers,DOCK-066,ACY9,2026-06-06,SSP,R,,
Schedule confirmed with vendor for next Tuesday 0900.
,P.Green,2026-06-01,Dock Systems,Dock Systems Inc +WO-1024,Repair bathroom faucet,PLUMB-012,BOS1,2026-06-07,SSP,IP,,
WO schedule confirmed with vendor. Technician on site 6/9.
,Q.Baker,2026-06-02,Metro Plumbing,Metro Plumbing Inc +WO-1025,Recalibrate HVAC thermostat,HVAC-033,ABQ5,2026-06-08,RME,H,SCHEDULING,,R.Adams,2026-06-03,Acme HVAC,Acme HVAC Services +WO-1026,Fix overhead door,DOOR-019,ACY9,2026-06-09,SSP,IP,,
Uplift request submitted pending management approval.
,S.Nelson,2026-06-04,Door Pro,Door Pro LLC +WO-1027,Replace signage,SIGN-004,BOS1,2026-06-10,BBM,IP,,,,2026-06-05,Sign Works,Sign Works Co +WO-1028,Inspect conveyor belt,CONV-022,ABQ5,2026-06-11,RME,IP,,,T.Carter,2026-06-06,Belt Tech,Belt Tech Services +WO-1029,Lubricate dock equipment,DOCK-077,ACY9,2026-06-12,SSP,R,,,U.Mitchell,2026-06-07,Dock Systems,Dock Systems Inc +WO-1030,Weekly PM on generator,GEN-008,BOS1,2026-06-13,RME,IP,,
Weekly WO scheduled. Service reports required EOD Friday.
,V.Perez,2026-06-08,Power Gen LLC,Power Generation LLC diff --git a/tests/test_classify.py b/tests/test_classify.py index 4d8cff8..e3fdb03 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -9,9 +9,13 @@ Canonical fixture: ``~/Downloads/_documents/Sheet1-1.xlsx`` (347 rows, 13 cols). If the file is absent the export-driven tests skip. Run with the repo venv: ./.venv/bin/python -m pytest tests/test_classify.py -s -q + +CSV fixture (always present, runs in CI): ``tests/fixtures/sample_export.csv``. """ import collections +import csv +import re import sys from pathlib import Path @@ -30,10 +34,31 @@ COL_LAST_COMMENT = 8 FIXTURE = Path.home() / "Downloads" / "_documents" / "Sheet1-1.xlsx" +# Committed CSV fixture — always present, no skipif. +CSV_FIXTURE = Path(__file__).resolve().parent / "fixtures" / "sample_export.csv" + # Max acceptable deterministic "Other" share before any AI. CLAUDE.md: the # two-axis model lands ~9% before Haiku; we hold the line at single digits. MAX_OTHER_PCT = 10.0 +# Threshold for the committed CSV fixture. Measured deterministic Other% on +# the synthetic fixture: 7.41% (2 of 27 classified rows). Threshold is set +# with headroom but still comfortably single-digit. +CSV_FIXTURE_MAX_OTHER_PCT = 9.0 + + +# --------------------------------------------------------------------------- +# CSV loader helper (yields column-indexed tuples like openpyxl row values) +# --------------------------------------------------------------------------- + + +def _load_csv_rows(path: Path) -> list[tuple]: + """Read a 13-column CSV export; return data rows as tuples (header skipped).""" + with path.open(newline="") as fh: + reader = csv.reader(fh) + rows = list(reader) + return [tuple(r) for r in rows[1:]] # drop header row + def test_classification_constants_well_formed(): assert "3rd Escalation" in classify.ESCALATION_CATEGORIES @@ -244,3 +269,163 @@ def test_known_fixture_rows_in_export(): report_completion_mismatch[COL_LAST_COMMENT], ) assert mm is not None + + +# --------------------------------------------------------------------------- +# CSV fixture tests — always run (no skipif), so the quality gate fires in CI. +# --------------------------------------------------------------------------- + + +def test_other_share_against_csv_fixture(capsys): + """Classification quality gate against the committed synthetic CSV fixture. + + Measured deterministic Other%: 7.41% (2/27). Threshold: 9.0%. + This test runs unconditionally in CI. + """ + rows = _load_csv_rows(CSV_FIXTURE) + + dist: collections.Counter = collections.Counter() + mismatches = 0 + classified = 0 + blank = 0 + other_samples: list[str] = [] + + for row in rows: + comment = row[COL_LAST_COMMENT] if len(row) > COL_LAST_COMMENT else "" + # Blank-comment rows are excluded from the classified total by design. + if comment is None or str(comment).strip() == "": + blank += 1 + continue + classified += 1 + category, mismatch = classify.classify( + row[COL_WO_STATUS], row[COL_HOLD_REASON], comment + ) + dist[category] += 1 + if mismatch: + mismatches += 1 + if category == "Other" and len(other_samples) < 20: + other_samples.append(classify.strip_html(comment)[:90]) + + other = dist["Other"] + other_pct = other * 100.0 / classified if classified else 0.0 + + with capsys.disabled(): + print(f"\n=== APM classifier smoke test (CSV fixture): {CSV_FIXTURE.name} ===") + print( + f"rows={len(rows)} classified={classified} " + f"blank-excluded={blank} (blank-comment rows excluded from the total)" + ) + print("--- category distribution ---") + for cat, count in dist.most_common(): + print(f" {count:4d} {count * 100.0 / classified:5.1f}% {cat}") + print(f"--- Other: {other} ({other_pct:.2f}%) ---") + for sample in other_samples: + print(f" [Other] {sample}") + print(f"--- mismatches flagged: {mismatches} ---") + + assert classified > 0, "CSV fixture produced no classified rows" + assert other_pct <= CSV_FIXTURE_MAX_OTHER_PCT, ( + f"deterministic Other {other_pct:.2f}% exceeds {CSV_FIXTURE_MAX_OTHER_PCT}% — " + "the two-axis ladder regressed against the committed fixture" + ) + # Fixture must exercise mismatch detection (WO-1012: REPORT hold + performed task). + assert mismatches >= 1, "CSV fixture should contain at least one mismatch row" + + +def test_known_rows_in_csv_fixture(): + """Anchor checks on the synthetic CSV fixture — runs unconditionally in CI.""" + rows = _load_csv_rows(CSV_FIXTURE) + + by_status: collections.defaultdict = collections.defaultdict(list) + schedule_confirmed_row = None + report_completion_mismatch = None + third_esc_rows: list[tuple] = [] + cancelled_rows: list[tuple] = [] + structured_report_rows: list[tuple] = [] + structured_scheduling_rows: list[tuple] = [] + + for row in rows: + comment = row[COL_LAST_COMMENT] if len(row) > COL_LAST_COMMENT else "" + if comment is None or str(comment).strip() == "": + continue + text = classify.strip_html(comment) + by_status[row[COL_WO_STATUS]].append(row) + + if ( + schedule_confirmed_row is None + and "schedule confirmed with vendor" in text.lower() + and not (row[COL_HOLD_REASON] or "").strip() + ): + schedule_confirmed_row = row + if ( + report_completion_mismatch is None + and (row[COL_HOLD_REASON] or "").strip().upper() == "REPORT" + and "performed task" in text.lower() + ): + report_completion_mismatch = row + if re.search(r"\b3rd\b.*\battempt\b", text.lower()): + third_esc_rows.append(row) + if row[COL_WO_STATUS] == "RCAN": + cancelled_rows.append(row) + # Structured-only: HTML-wrapped empty comment (strips to "") with REPORT hold. + if (row[COL_HOLD_REASON] or "").strip().upper() == "REPORT" and text == "": + structured_report_rows.append(row) + if (row[COL_HOLD_REASON] or "").strip().upper() == "SCHEDULING" and text == "": + structured_scheduling_rows.append(row) + + # "WO schedule confirmed with vendor" → Schedule Confirmed. + assert schedule_confirmed_row is not None, "fixture lacks a schedule-confirmed row" + cat, _ = classify.classify( + schedule_confirmed_row[COL_WO_STATUS], + schedule_confirmed_row[COL_HOLD_REASON], + schedule_confirmed_row[COL_LAST_COMMENT], + ) + assert cat == "Schedule Confirmed", f"expected Schedule Confirmed, got {cat!r}" + + # RCAN rows → Cancelled. + assert cancelled_rows, "fixture lacks an RCAN row" + cat, _ = classify.classify( + cancelled_rows[0][COL_WO_STATUS], + cancelled_rows[0][COL_HOLD_REASON], + cancelled_rows[0][COL_LAST_COMMENT], + ) + assert cat == "Cancelled", f"expected Cancelled, got {cat!r}" + + # 3rd Escalation rows are present and classify correctly. + assert third_esc_rows, "fixture lacks a 3rd-escalation row" + cat, _ = classify.classify( + third_esc_rows[0][COL_WO_STATUS], + third_esc_rows[0][COL_HOLD_REASON], + third_esc_rows[0][COL_LAST_COMMENT], + ) + assert cat == "3rd Escalation", f"expected 3rd Escalation, got {cat!r}" + + # Structured-only REPORT rows → Report / Docs Needed. + assert structured_report_rows, "fixture lacks a structured-only REPORT hold row" + cat, _ = classify.classify( + structured_report_rows[0][COL_WO_STATUS], + structured_report_rows[0][COL_HOLD_REASON], + structured_report_rows[0][COL_LAST_COMMENT], + ) + assert cat == "Report / Docs Needed", f"expected Report / Docs Needed, got {cat!r}" + + # Structured-only SCHEDULING rows → Awaiting Scheduling. + assert structured_scheduling_rows, "fixture lacks a structured-only SCHEDULING row" + cat, _ = classify.classify( + structured_scheduling_rows[0][COL_WO_STATUS], + structured_scheduling_rows[0][COL_HOLD_REASON], + structured_scheduling_rows[0][COL_LAST_COMMENT], + ) + assert cat == "Awaiting Scheduling", f"expected Awaiting Scheduling, got {cat!r}" + + # REPORT-hold row with completion comment → mismatch surfaced. + assert report_completion_mismatch is not None, ( + "fixture lacks a REPORT-hold row with a completion comment (mismatch case)" + ) + _, mm = classify.classify( + report_completion_mismatch[COL_WO_STATUS], + report_completion_mismatch[COL_HOLD_REASON], + report_completion_mismatch[COL_LAST_COMMENT], + ) + assert mm is not None, "expected a mismatch reason for WO-1012 but got None" + assert "REPORT" in mm, f"mismatch reason should mention REPORT hold: {mm!r}"