mirror of
https://github.com/Sea-Haven-Industries/apm-wo-analysis.git
synced 2026-09-30 03:03:14 +00:00
Add synthetic export fixture and run classification quality gate in CI
This commit is contained in:
Adam Moussa
parent
1a126c8179
commit
817e1f6a74
2 changed files with 216 additions and 0 deletions
31
tests/fixtures/sample_export.csv
vendored
Normal file
31
tests/fixtures/sample_export.csv
vendored
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
WO Number,WO Description,Equipment Code,Organization,Due Date,Department,WO Status,Hold Reason,Last Comment,Last Comment By,Last Comment Date,Contractor,Contractor Description
|
||||
WO-1001,Repair HVAC unit at dock 3,HVAC-001,ABQ5,2026-05-15,SSP,H,REPORT,<html><div>3rd attempt process for schedule confirmation. Vendor please confirm schedule start date and proceed.</div></html>,J.Smith,2026-05-10,Acme HVAC,Acme HVAC Services
|
||||
WO-1002,Replace lighting ballast,LIGHT-042,ACY9,2026-05-16,RME,H,SCHEDULING,<html><div>2nd attempt process for schedule confirmation. Please provide a confirmed date for this work order.</div></html>,T.Jones,2026-05-11,Bright Electric,Bright Electric LLC
|
||||
WO-1003,Fix water leak in restroom,PLUMB-007,BOS1,2026-05-17,SSP,IP,,<html><div>1st escalation sent to vendor. Awaiting response from contractor regarding scheduling.</div></html>,M.Davis,2026-05-12,Metro Plumbing,Metro Plumbing Inc
|
||||
WO-1004,Inspect fire suppression system,FIRE-015,ABQ5,2026-05-18,SSP,R,,<html><div>SIM ticket TT-12345678 opened for this work order. Tracking in Amazon SIM system.</div></html>,K.Wilson,2026-05-13,FireSafe Co,FireSafe Company
|
||||
WO-1005,Service generator unit,GEN-003,ACY9,2026-05-19,RME,R,,<html><div>Site tech reported vendor was a no show for Tuesday. Rescheduling required.</div></html>,P.Brown,2026-05-14,Power Gen LLC,Power Generation LLC
|
||||
WO-1006,Calibrate dock door sensors,DOCK-022,BOS1,2026-05-20,SSP,IP,,<html><div>WO schedule confirmed with vendor for 5/22. All parties notified.</div></html>,L.Garcia,2026-05-15,Dock Systems,Dock Systems Inc
|
||||
WO-1007,Weekly PM on conveyor belt,CONV-011,ABQ5,2026-05-21,RME,IP,,<html><div>Weekly WO scheduled. Service reports required EOD Friday per standard cadence.</div></html>,R.Martinez,2026-05-16,Belt Tech,Belt Tech Services
|
||||
WO-1008,Replace worn floor mats,FLOOR-009,ACY9,2026-05-22,BBM,H,REPORT,<html></html>,A.Thompson,2026-05-17,Clean Facility,Clean Facility Services
|
||||
WO-1009,Inspect sprinkler heads,FIRE-031,BOS1,2026-05-23,SSP,H,SCHEDULING,<html></html>,B.Anderson,2026-05-18,FireSafe Co,FireSafe Company
|
||||
WO-1010,Order replacement motor,MOTOR-005,ABQ5,2026-05-24,RME,H,VENDOR,<html></html>,C.Jackson,2026-05-19,Motor Supply,Motor Supply Co
|
||||
WO-1011,Cancel duplicate work order,WO-DUP-001,ACY9,2026-05-25,SSP,RCAN,,<html><div>WO cancelled, created in error. Duplicate of WO-1010.</div></html>,D.White,2026-05-20,N/A,N/A
|
||||
WO-1012,Perform task and close,TASK-088,BOS1,2026-05-26,RME,H,REPORT,<html><div>Vendor arrived and performed task. All work completed satisfactorily.</div></html>,E.Harris,2026-05-21,General Contractors,General Contractors LLC
|
||||
WO-1013,PM on roof HVAC,HVAC-099,ABQ5,2026-05-27,SSP,R,,<html><div>WO schedule confirmed with vendor for 5/29 start time 0800.</div></html>,F.Clark,2026-05-22,Acme HVAC,Acme HVAC Services
|
||||
WO-1014,Service dock bumpers weekly,DOCK-044,ACY9,2026-05-28,SSP,IP,,<html><div>Weekly WO scheduled. Service reports are required weekly per maintenance plan.</div></html>,G.Lewis,2026-05-23,Dock Systems,Dock Systems Inc
|
||||
WO-1015,Replace battery backup,UPS-002,BOS1,2026-05-29,RME,H,SCHEDULING,<html><div>2nd attempt process for schedule confirmation. Vendor has not responded to prior outreach.</div></html>,H.Robinson,2026-05-24,Power Gen LLC,Power Generation LLC
|
||||
WO-1016,Inspect emergency exits,EXIT-007,ABQ5,2026-05-30,SSP,R,,<html><div>3rd attempt process for schedule confirmation. Escalating to vendor management for response.</div></html>,I.Walker,2026-05-25,Safety First,Safety First Inc
|
||||
WO-1017,Fix broken dock seal,DOCK-055,ACY9,2026-05-31,SSP,IP,,<html><div>Vendor confirmed schedule for 6/2 arrival.</div></html>,J.Hall,2026-05-26,Dock Systems,Dock Systems Inc
|
||||
WO-1018,Service fire extinguishers,FIRE-022,BOS1,2026-06-01,SSP,RCAN,,<html><div>WO cancelled. Work was already completed under WO-1004.</div></html>,K.Young,2026-05-27,FireSafe Co,FireSafe Company
|
||||
WO-1019,PM on cooling tower,COOL-001,ABQ5,2026-06-02,RME,H,REPORT,<html></html>,L.Allen,2026-05-28,Cooling Tech,Cooling Tech LLC
|
||||
WO-1020,Inspect electrical panel,ELEC-018,ACY9,2026-06-03,RME,H,VENDOR,<html></html>,M.King,2026-05-29,Volt Electric,Volt Electric Services
|
||||
WO-1021,Replace worn casters on cart,CART-003,BOS1,2026-06-04,BBM,IP,,<html><div>Procurement to provide PO for parts order before work can begin.</div></html>,N.Wright,2026-05-30,General Contractors,General Contractors LLC
|
||||
WO-1022,Patch roof leak,ROOF-006,ABQ5,2026-06-05,SSP,IP,,<html><div>Copy.</div></html>,O.Scott,2026-05-31,Roof Masters,Roof Masters Inc
|
||||
WO-1023,Service dock bumpers,DOCK-066,ACY9,2026-06-06,SSP,R,,<html><div>Schedule confirmed with vendor for next Tuesday 0900.</div></html>,P.Green,2026-06-01,Dock Systems,Dock Systems Inc
|
||||
WO-1024,Repair bathroom faucet,PLUMB-012,BOS1,2026-06-07,SSP,IP,,<html><div>WO schedule confirmed with vendor. Technician on site 6/9.</div></html>,Q.Baker,2026-06-02,Metro Plumbing,Metro Plumbing Inc
|
||||
WO-1025,Recalibrate HVAC thermostat,HVAC-033,ABQ5,2026-06-08,RME,H,SCHEDULING,<html></html>,R.Adams,2026-06-03,Acme HVAC,Acme HVAC Services
|
||||
WO-1026,Fix overhead door,DOOR-019,ACY9,2026-06-09,SSP,IP,,<html><div>Uplift request submitted pending management approval.</div></html>,S.Nelson,2026-06-04,Door Pro,Door Pro LLC
|
||||
WO-1027,Replace signage,SIGN-004,BOS1,2026-06-10,BBM,IP,,,,2026-06-05,Sign Works,Sign Works Co
|
||||
WO-1028,Inspect conveyor belt,CONV-022,ABQ5,2026-06-11,RME,IP,,,T.Carter,2026-06-06,Belt Tech,Belt Tech Services
|
||||
WO-1029,Lubricate dock equipment,DOCK-077,ACY9,2026-06-12,SSP,R,,,U.Mitchell,2026-06-07,Dock Systems,Dock Systems Inc
|
||||
WO-1030,Weekly PM on generator,GEN-008,BOS1,2026-06-13,RME,IP,,<html><div>Weekly WO scheduled. Service reports required EOD Friday.</div></html>,V.Perez,2026-06-08,Power Gen LLC,Power Generation LLC
|
||||
|
Can't render this file because it has a wrong number of fields in line 12.
|
|
|
@ -9,9 +9,13 @@ Canonical fixture: ``~/Downloads/_documents/Sheet1-1.xlsx`` (347 rows, 13 cols).
|
|||
If the file is absent the export-driven tests skip. Run with the repo venv:
|
||||
|
||||
./.venv/bin/python -m pytest tests/test_classify.py -s -q
|
||||
|
||||
CSV fixture (always present, runs in CI): ``tests/fixtures/sample_export.csv``.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import csv
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -30,10 +34,31 @@ COL_LAST_COMMENT = 8
|
|||
|
||||
FIXTURE = Path.home() / "Downloads" / "_documents" / "Sheet1-1.xlsx"
|
||||
|
||||
# Committed CSV fixture — always present, no skipif.
|
||||
CSV_FIXTURE = Path(__file__).resolve().parent / "fixtures" / "sample_export.csv"
|
||||
|
||||
# Max acceptable deterministic "Other" share before any AI. CLAUDE.md: the
|
||||
# two-axis model lands ~9% before Haiku; we hold the line at single digits.
|
||||
MAX_OTHER_PCT = 10.0
|
||||
|
||||
# Threshold for the committed CSV fixture. Measured deterministic Other% on
|
||||
# the synthetic fixture: 7.41% (2 of 27 classified rows). Threshold is set
|
||||
# with headroom but still comfortably single-digit.
|
||||
CSV_FIXTURE_MAX_OTHER_PCT = 9.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSV loader helper (yields column-indexed tuples like openpyxl row values)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_csv_rows(path: Path) -> list[tuple]:
|
||||
"""Read a 13-column CSV export; return data rows as tuples (header skipped)."""
|
||||
with path.open(newline="") as fh:
|
||||
reader = csv.reader(fh)
|
||||
rows = list(reader)
|
||||
return [tuple(r) for r in rows[1:]] # drop header row
|
||||
|
||||
|
||||
def test_classification_constants_well_formed():
|
||||
assert "3rd Escalation" in classify.ESCALATION_CATEGORIES
|
||||
|
|
@ -244,3 +269,163 @@ def test_known_fixture_rows_in_export():
|
|||
report_completion_mismatch[COL_LAST_COMMENT],
|
||||
)
|
||||
assert mm is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSV fixture tests — always run (no skipif), so the quality gate fires in CI.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_other_share_against_csv_fixture(capsys):
|
||||
"""Classification quality gate against the committed synthetic CSV fixture.
|
||||
|
||||
Measured deterministic Other%: 7.41% (2/27). Threshold: 9.0%.
|
||||
This test runs unconditionally in CI.
|
||||
"""
|
||||
rows = _load_csv_rows(CSV_FIXTURE)
|
||||
|
||||
dist: collections.Counter = collections.Counter()
|
||||
mismatches = 0
|
||||
classified = 0
|
||||
blank = 0
|
||||
other_samples: list[str] = []
|
||||
|
||||
for row in rows:
|
||||
comment = row[COL_LAST_COMMENT] if len(row) > COL_LAST_COMMENT else ""
|
||||
# Blank-comment rows are excluded from the classified total by design.
|
||||
if comment is None or str(comment).strip() == "":
|
||||
blank += 1
|
||||
continue
|
||||
classified += 1
|
||||
category, mismatch = classify.classify(
|
||||
row[COL_WO_STATUS], row[COL_HOLD_REASON], comment
|
||||
)
|
||||
dist[category] += 1
|
||||
if mismatch:
|
||||
mismatches += 1
|
||||
if category == "Other" and len(other_samples) < 20:
|
||||
other_samples.append(classify.strip_html(comment)[:90])
|
||||
|
||||
other = dist["Other"]
|
||||
other_pct = other * 100.0 / classified if classified else 0.0
|
||||
|
||||
with capsys.disabled():
|
||||
print(f"\n=== APM classifier smoke test (CSV fixture): {CSV_FIXTURE.name} ===")
|
||||
print(
|
||||
f"rows={len(rows)} classified={classified} "
|
||||
f"blank-excluded={blank} (blank-comment rows excluded from the total)"
|
||||
)
|
||||
print("--- category distribution ---")
|
||||
for cat, count in dist.most_common():
|
||||
print(f" {count:4d} {count * 100.0 / classified:5.1f}% {cat}")
|
||||
print(f"--- Other: {other} ({other_pct:.2f}%) ---")
|
||||
for sample in other_samples:
|
||||
print(f" [Other] {sample}")
|
||||
print(f"--- mismatches flagged: {mismatches} ---")
|
||||
|
||||
assert classified > 0, "CSV fixture produced no classified rows"
|
||||
assert other_pct <= CSV_FIXTURE_MAX_OTHER_PCT, (
|
||||
f"deterministic Other {other_pct:.2f}% exceeds {CSV_FIXTURE_MAX_OTHER_PCT}% — "
|
||||
"the two-axis ladder regressed against the committed fixture"
|
||||
)
|
||||
# Fixture must exercise mismatch detection (WO-1012: REPORT hold + performed task).
|
||||
assert mismatches >= 1, "CSV fixture should contain at least one mismatch row"
|
||||
|
||||
|
||||
def test_known_rows_in_csv_fixture():
|
||||
"""Anchor checks on the synthetic CSV fixture — runs unconditionally in CI."""
|
||||
rows = _load_csv_rows(CSV_FIXTURE)
|
||||
|
||||
by_status: collections.defaultdict = collections.defaultdict(list)
|
||||
schedule_confirmed_row = None
|
||||
report_completion_mismatch = None
|
||||
third_esc_rows: list[tuple] = []
|
||||
cancelled_rows: list[tuple] = []
|
||||
structured_report_rows: list[tuple] = []
|
||||
structured_scheduling_rows: list[tuple] = []
|
||||
|
||||
for row in rows:
|
||||
comment = row[COL_LAST_COMMENT] if len(row) > COL_LAST_COMMENT else ""
|
||||
if comment is None or str(comment).strip() == "":
|
||||
continue
|
||||
text = classify.strip_html(comment)
|
||||
by_status[row[COL_WO_STATUS]].append(row)
|
||||
|
||||
if (
|
||||
schedule_confirmed_row is None
|
||||
and "schedule confirmed with vendor" in text.lower()
|
||||
and not (row[COL_HOLD_REASON] or "").strip()
|
||||
):
|
||||
schedule_confirmed_row = row
|
||||
if (
|
||||
report_completion_mismatch is None
|
||||
and (row[COL_HOLD_REASON] or "").strip().upper() == "REPORT"
|
||||
and "performed task" in text.lower()
|
||||
):
|
||||
report_completion_mismatch = row
|
||||
if re.search(r"\b3rd\b.*\battempt\b", text.lower()):
|
||||
third_esc_rows.append(row)
|
||||
if row[COL_WO_STATUS] == "RCAN":
|
||||
cancelled_rows.append(row)
|
||||
# Structured-only: HTML-wrapped empty comment (strips to "") with REPORT hold.
|
||||
if (row[COL_HOLD_REASON] or "").strip().upper() == "REPORT" and text == "":
|
||||
structured_report_rows.append(row)
|
||||
if (row[COL_HOLD_REASON] or "").strip().upper() == "SCHEDULING" and text == "":
|
||||
structured_scheduling_rows.append(row)
|
||||
|
||||
# "WO schedule confirmed with vendor" → Schedule Confirmed.
|
||||
assert schedule_confirmed_row is not None, "fixture lacks a schedule-confirmed row"
|
||||
cat, _ = classify.classify(
|
||||
schedule_confirmed_row[COL_WO_STATUS],
|
||||
schedule_confirmed_row[COL_HOLD_REASON],
|
||||
schedule_confirmed_row[COL_LAST_COMMENT],
|
||||
)
|
||||
assert cat == "Schedule Confirmed", f"expected Schedule Confirmed, got {cat!r}"
|
||||
|
||||
# RCAN rows → Cancelled.
|
||||
assert cancelled_rows, "fixture lacks an RCAN row"
|
||||
cat, _ = classify.classify(
|
||||
cancelled_rows[0][COL_WO_STATUS],
|
||||
cancelled_rows[0][COL_HOLD_REASON],
|
||||
cancelled_rows[0][COL_LAST_COMMENT],
|
||||
)
|
||||
assert cat == "Cancelled", f"expected Cancelled, got {cat!r}"
|
||||
|
||||
# 3rd Escalation rows are present and classify correctly.
|
||||
assert third_esc_rows, "fixture lacks a 3rd-escalation row"
|
||||
cat, _ = classify.classify(
|
||||
third_esc_rows[0][COL_WO_STATUS],
|
||||
third_esc_rows[0][COL_HOLD_REASON],
|
||||
third_esc_rows[0][COL_LAST_COMMENT],
|
||||
)
|
||||
assert cat == "3rd Escalation", f"expected 3rd Escalation, got {cat!r}"
|
||||
|
||||
# Structured-only REPORT rows → Report / Docs Needed.
|
||||
assert structured_report_rows, "fixture lacks a structured-only REPORT hold row"
|
||||
cat, _ = classify.classify(
|
||||
structured_report_rows[0][COL_WO_STATUS],
|
||||
structured_report_rows[0][COL_HOLD_REASON],
|
||||
structured_report_rows[0][COL_LAST_COMMENT],
|
||||
)
|
||||
assert cat == "Report / Docs Needed", f"expected Report / Docs Needed, got {cat!r}"
|
||||
|
||||
# Structured-only SCHEDULING rows → Awaiting Scheduling.
|
||||
assert structured_scheduling_rows, "fixture lacks a structured-only SCHEDULING row"
|
||||
cat, _ = classify.classify(
|
||||
structured_scheduling_rows[0][COL_WO_STATUS],
|
||||
structured_scheduling_rows[0][COL_HOLD_REASON],
|
||||
structured_scheduling_rows[0][COL_LAST_COMMENT],
|
||||
)
|
||||
assert cat == "Awaiting Scheduling", f"expected Awaiting Scheduling, got {cat!r}"
|
||||
|
||||
# REPORT-hold row with completion comment → mismatch surfaced.
|
||||
assert report_completion_mismatch is not None, (
|
||||
"fixture lacks a REPORT-hold row with a completion comment (mismatch case)"
|
||||
)
|
||||
_, mm = classify.classify(
|
||||
report_completion_mismatch[COL_WO_STATUS],
|
||||
report_completion_mismatch[COL_HOLD_REASON],
|
||||
report_completion_mismatch[COL_LAST_COMMENT],
|
||||
)
|
||||
assert mm is not None, "expected a mismatch reason for WO-1012 but got None"
|
||||
assert "REPORT" in mm, f"mismatch reason should mention REPORT hold: {mm!r}"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue