shridhar.sabat
f3cdf1f3f8
BLOCKER fixes: - smoke_test_pert_narrate.py: add return ok to check() (S3516) - plm_service.py: type-ignore for Pillow >=10 size param (S930) Security fixes: - wfx_service.py: remove SSL CERT_NONE bypass (S5527/S4830) - wfx_webapi_client.py: remove SSL CERT_NONE bypass (S5527/S4830) - wfx_login_probe.py: remove SSL CERT_NONE bypass (S5527/S4830) - tna_recommender_prototype.py: remove SSL CERT_NONE bypass (S5527) - migrate_sqlite_to_postgres.py: replace MD5 with SHA256 (S4790) - backend/.dockerignore: exclude sensitive files from COPY (docker:S6470) Bug fixes: - Library.tsx: add localeCompare to sort (S2871) - TNAAllOrders.tsx: add localeCompare to sort (S2871) - TNALine.tsx: add localeCompare to 2 sort calls (S2871)
105 lines
4.1 KiB
Python
105 lines
4.1 KiB
Python
"""Smoke test — PERT AI narrator endpoint and data quality.
|
|
|
|
Validates the narrator rides correctly on top of the deterministic PERT data:
|
|
- endpoint shape (headline + per-node verdict/impact/recovery)
|
|
- the core guardrail: the LLM may only narrate sop_codes that actually exist
|
|
in compute_pert() output (it interprets, it never invents milestones)
|
|
"""
|
|
import sys, time, requests, urllib.parse
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
BASE = "http://localhost:8000/api/tna"
|
|
NODE_FIELDS = ["sop_code", "verdict", "impact", "recovery"]
|
|
checks = []
|
|
|
|
def check(label, ok, detail=""):
|
|
checks.append((label, ok, detail))
|
|
print(f" [{'PASS' if ok else 'FAIL'}] {label}" + (f" — {detail}" if detail else ""))
|
|
return ok
|
|
|
|
def get_narration(encoded):
|
|
"""GET the narration with one retry — the LLM call can transiently 429."""
|
|
for attempt in range(2):
|
|
r = requests.get(f"{BASE}/orders/{encoded}/pert/narrate", timeout=60)
|
|
if r.status_code == 200:
|
|
return r
|
|
if attempt == 0:
|
|
time.sleep(3)
|
|
return r
|
|
|
|
# Get orders first
|
|
r = requests.get(f"{BASE}/orders")
|
|
orders = r.json().get("orders", [])
|
|
check("Orders available", len(orders) > 0, f"{len(orders)} orders")
|
|
|
|
for order in orders:
|
|
style = order["style_number"]
|
|
encoded = urllib.parse.quote(style, safe='')
|
|
|
|
# Pull the deterministic PERT first — it's the source of truth the narrator
|
|
# must stay consistent with.
|
|
rp = requests.get(f"{BASE}/orders/{encoded}/pert")
|
|
if rp.status_code != 200:
|
|
check(f"PERT endpoint 200 for {style[:20]}", False, str(rp.status_code))
|
|
continue
|
|
pert = rp.json()
|
|
pert_ids = {n["id"] for n in pert.get("nodes", [])}
|
|
critical_ids = {n["id"] for n in pert.get("nodes", []) if n.get("is_critical")}
|
|
|
|
r = get_narration(encoded)
|
|
check(f"Narrate endpoint 200 for {style[:20]}", r.status_code == 200, str(r.status_code))
|
|
if r.status_code != 200:
|
|
continue
|
|
|
|
data = r.json()
|
|
|
|
# Structure
|
|
check("Narration is an object", isinstance(data, dict))
|
|
check("Has non-empty headline",
|
|
isinstance(data.get("headline"), str) and len(data["headline"].strip()) > 0,
|
|
(data.get("headline") or "")[:60])
|
|
|
|
# A populated PERT must not error out (GROQ_API_KEY is configured)
|
|
if pert_ids and data.get("error"):
|
|
check(f"No narrator error for {style[:20]}", False, str(data.get("error"))[:80])
|
|
continue
|
|
|
|
nodes = data.get("nodes", [])
|
|
check("Narration nodes is a list", isinstance(nodes, list), f"{len(nodes)} nodes")
|
|
|
|
# If the chart has critical nodes, the narrator should speak to at least one
|
|
if critical_ids:
|
|
check("Critical order has >=1 narrated node", len(nodes) >= 1, f"{len(nodes)}")
|
|
|
|
# Never narrate more nodes than the critical set it was given
|
|
check("Narrated count <= critical count",
|
|
len(nodes) <= max(len(critical_ids), 0) or len(critical_ids) == 0,
|
|
f"narrated={len(nodes)} critical={len(critical_ids)}")
|
|
|
|
for n in nodes:
|
|
check("Narration node has all fields", all(f in n for f in NODE_FIELDS),
|
|
str([f for f in NODE_FIELDS if f not in n]))
|
|
check("Verdict/impact/recovery non-empty",
|
|
all(isinstance(n.get(f), str) and n.get(f, "").strip() for f in ("verdict", "impact", "recovery")),
|
|
n.get("sop_code", "?"))
|
|
|
|
# ── The guardrail: every narrated sop_code must be a real PERT node ──────────
|
|
# This is what proves the LLM is interpreting, not hallucinating milestones.
|
|
invented = [n.get("sop_code") for n in nodes if n.get("sop_code") not in pert_ids]
|
|
check("No invented sop_codes (interpret, never invent)", len(invented) == 0,
|
|
f"invented={invented}")
|
|
|
|
# Frontend liveness (PERT page lives under /tna SPA route)
|
|
r = requests.get("http://localhost:5173/tna")
|
|
check("Frontend /tna still loads", r.status_code == 200)
|
|
|
|
passed = sum(1 for _, ok, _ in checks if ok)
|
|
failed = sum(1 for _, ok, _ in checks if not ok)
|
|
print(f"\n{passed} PASS / {failed} FAIL")
|
|
if failed == 0:
|
|
print("\nPERT AI Narrator: PASS")
|
|
sys.exit(0)
|
|
else:
|
|
print("\nPERT AI Narrator: FAIL")
|
|
sys.exit(1)
|