Testing with AI, and testing AI features

You are a senior QA engineer. Here is a requirement for an API.

--- requirement ---
[paste requirement or OpenAPI snippet]
---

List test cases as a table with columns: ID, Scenario, Input, Expected result, Type (positive, negative, boundary, security). Include at least 5 negative and 5 boundary cases. Then list the questions you would ask the product owner because the requirement is unclear.

Here is a FastAPI endpoint and its Pydantic model. Write pytest tests using FastAPI's TestClient.

[paste main.py]

Cover: the happy path, every validation rule (one failing test per rule), boundary values, a missing body, a wrong type, and the not-found case. Use pytest.mark.parametrize where several cases share logic. Each test name must say what behaviour it checks. Do not change main.py.

def test_create_order_returns_total(make_order):
    response = make_order()
    assert response.status_code == 201
    assert "total" in response.json()

def test_create_order_total_is_price_times_quantity(make_order):
    response = make_order(items=[{"item_id": "espresso", "qty": 2}])
    assert response.status_code == 201
    assert response.json()["total"] == 240  # 2 x Rs 120, from the menu

import json

ALLOWED = {"praise", "complaint", "question"}


def classify_feedback(text, llm):
    """Ask a model to label customer feedback. `llm` is any function: prompt -> str."""
    prompt = (
        "Classify this coffee shop feedback as exactly one of: praise, complaint, question.\n"
        'Reply with JSON only, like {"label": "praise", "confidence": 0.9}.\n\n'
        f"Feedback: {text}"
    )
    raw = llm(prompt)
    try:
        data = json.loads(raw)
        label = data["label"]
        confidence = float(data["confidence"])
    except (ValueError, KeyError, TypeError):
        return {"label": "unknown", "confidence": 0.0}
    if label not in ALLOWED or not 0 <= confidence <= 1:
        return {"label": "unknown", "confidence": 0.0}
    return {"label": label, "confidence": confidence}

import pytest

from feedback import classify_feedback


# Part 1: the code AROUND the model is normal, deterministic code. Test it with a fake model.
def fake_llm(reply):
    return lambda prompt: reply


def test_valid_reply_is_parsed():
    result = classify_feedback("Lovely cold brew!", fake_llm('{"label": "praise", "confidence": 0.93}'))
    assert result == {"label": "praise", "confidence": 0.93}


@pytest.mark.parametrize(
    "bad_reply",
    [
        "Sure! The label is praise.",
        "{not json",
        '{"label": "angry", "confidence": 0.9}',
        '{"label": "praise", "confidence": 7}',
        '{"label": "praise"}',
        "null",
    ],
)
def test_bad_model_output_becomes_unknown(bad_reply):
    assert classify_feedback("anything", fake_llm(bad_reply))["label"] == "unknown"


def test_prompt_contains_the_feedback_text():
    seen = []
    classify_feedback("Is the croissant vegan?", lambda p: seen.append(p) or "{}")
    assert "Is the croissant vegan?" in seen[0]

from feedback import classify_feedback

# A tiny golden set: real-looking inputs with the label a human agreed on.
GOLDEN_SET = [
    ("The cold brew was smooth and the staff were lovely.", "praise"),
    ("Best filter coffee in Indiranagar, thank you!", "praise"),
    ("My cappuccino was cold and the order took 40 minutes.", "complaint"),
    ("I was charged Rs 220 but the bill said 180.", "complaint"),
    ("Do you have oat milk?", "question"),
    ("Is the banana bread egg-free?", "question"),
]
PASS_RATE = 0.8


def real_llm(prompt):
    # Replace this body with a call to your model provider and return the reply text.
    raise NotImplementedError("plug in your model call here")


def run_eval(llm):
    wrong = []
    for text, expected in GOLDEN_SET:
        got = classify_feedback(text, llm)["label"]
        if got != expected:
            wrong.append((text, expected, got))
    rate = 1 - len(wrong) / len(GOLDEN_SET)
    for text, expected, got in wrong:
        print(f"WRONG: {text!r} expected={expected} got={got}")
    print(f"pass rate: {rate:.0%} (needs {PASS_RATE:.0%})")
    return rate >= PASS_RATE


if __name__ == "__main__":
    raise SystemExit(0 if run_eval(real_llm) else 1)