Project: coffee shop menu assistant with RAG

Create a JSON file menu.json for a coffee shop with these 8 items: [paste the menu: id, name, category, price in rupees]. Add for each item a one-sentence description and an "allergens" string (use "none" if there are no common allergens). Return only JSON.

[
  {"id": "espresso", "name": "Espresso", "category": "coffee", "price": 120,
   "description": "A short, strong shot of coffee made from our house blend.", "allergens": "none"},
  {"id": "cappuccino", "name": "Cappuccino", "category": "coffee", "price": 180,
   "description": "Espresso with steamed milk and a thick layer of foam.", "allergens": "milk"},
  {"id": "cold-brew", "name": "Cold Brew", "category": "coffee", "price": 220,
   "description": "Coffee steeped in cold water for 18 hours. Smooth, low in bitterness, served over ice.", "allergens": "none"},
  {"id": "filter-coffee", "name": "Filter Coffee", "category": "coffee", "price": 80,
   "description": "South Indian style decoction coffee with hot milk and sugar.", "allergens": "milk"},
  {"id": "masala-chai", "name": "Masala Chai", "category": "tea", "price": 90,
   "description": "Black tea simmered with milk, ginger and cardamom.", "allergens": "milk"},
  {"id": "croissant", "name": "Butter Croissant", "category": "bakery", "price": 150,
   "description": "Flaky, buttery croissant baked fresh every morning.", "allergens": "gluten, milk"},
  {"id": "banana-bread", "name": "Banana Bread", "category": "bakery", "price": 130,
   "description": "Moist banana bread with walnuts.", "allergens": "gluten, eggs, nuts"},
  {"id": "veg-sandwich", "name": "Veg Grilled Sandwich", "category": "food", "price": 160,
   "description": "Grilled sandwich with cheese, tomato, onion and capsicum.", "allergens": "gluten, milk"}
]

Opening hours: Brew & Bean is open every day from 8 am to 10 pm.

Pickup only: we do not deliver. Order on the website, then collect your order from the counter at your pickup time.

Order status: an order goes from received to preparing to ready, and then to picked up. You can cancel only while it is still in the received stage.

Milk alternatives: we can make any coffee or chai with oat milk for an extra Rs 40.

Payments: we accept UPI, cards and cash at the counter.

import json
from pathlib import Path

HERE = Path(__file__).parent


def load_documents():
    """Return a list of documents: {id, text, meta}. One document per menu item
    or FAQ paragraph, so each is small and about one thing."""
    docs = []
    for item in json.loads((HERE / "menu.json").read_text(encoding="utf-8")):
        text = (
            f"{item['name']} ({item['category']}), Rs {item['price']}. "
            f"{item['description']} Allergens: {item['allergens']}."
        )
        docs.append({"id": f"menu:{item['id']}", "text": text,
                     "meta": {"type": "menu", "category": item["category"]}})
    faq = (HERE / "faq.txt").read_text(encoding="utf-8")
    for i, para in enumerate(p.strip() for p in faq.split("\n\n") if p.strip()):
        docs.append({"id": f"faq:{i}", "text": para, "meta": {"type": "faq", "category": None}})
    return docs


if __name__ == "__main__":
    for d in load_documents():
        print(d["id"], "|", d["text"][:70])

python data.py

I'm building RAG over a small coffee shop menu (8 items) and 5 FAQ paragraphs. I plan to use BM25 plus embeddings merged with reciprocal rank fusion, and a metadata filter on category. List 5 realistic customer questions that would break vector-only search and 5 that would break keyword-only search. Be specific.

import re

import numpy as np
from google import genai
from google.genai import types
from rank_bm25 import BM25Okapi

from data import HERE, load_documents

client = genai.Client()  # reads GEMINI_API_KEY
EMBED_MODEL = "gemini-embedding-001"
INDEX_FILE = HERE / "index.npy"
DOCS = load_documents()


def tokenize(text):
    return re.findall(r"\w+", text.lower())


def embed(texts, task_type):
    resp = client.models.embed_content(
        model=EMBED_MODEL, contents=texts,
        config=types.EmbedContentConfig(task_type=task_type))
    arr = np.array([e.values for e in resp.embeddings], dtype=np.float32)
    return arr / np.linalg.norm(arr, axis=1, keepdims=True)


def load_matrix():
    """Embed the documents once and cache the vectors on disk."""
    if INDEX_FILE.exists():
        matrix = np.load(INDEX_FILE)
        if len(matrix) == len(DOCS):
            return matrix
    matrix = embed([d["text"] for d in DOCS], "RETRIEVAL_DOCUMENT")
    np.save(INDEX_FILE, matrix)
    return matrix


MATRIX = load_matrix()
BM25 = BM25Okapi([tokenize(d["text"]) for d in DOCS])


def rrf(rankings, k=60):
    scores = {}
    for ranking in rankings:
        for pos, i in enumerate(ranking, start=1):
            scores[i] = scores.get(i, 0) + 1 / (k + pos)
    return sorted(scores, key=scores.get, reverse=True)


def search(question, k=4, category=None):
    """Hybrid search. If `category` is given, only that menu category is searched."""
    allowed = [i for i, d in enumerate(DOCS)
               if category is None or d["meta"]["category"] == category]
    kw = BM25.get_scores(tokenize(question))
    q = embed([question], "RETRIEVAL_QUERY")[0]
    vec = MATRIX @ q
    kw_rank = sorted(allowed, key=lambda i: -kw[i])[:10]
    vec_rank = sorted(allowed, key=lambda i: -vec[i])[:10]
    return [DOCS[i] for i in rrf([kw_rank, vec_rank])[:k]]

Here is my system prompt for a coffee shop ordering assistant that answers from numbered sources: [paste SYSTEM]. Find 5 ways a customer or the model could make it misbehave (for example inventing a price, answering a medical allergy question with false certainty, following instructions hidden in a question). Suggest a one-line fix for each.

import re

from google.genai import types

from search import client, search

GEN_MODEL = "gemini-flash-latest"  # check current model names on ai.google.dev

SYSTEM = """You are the ordering assistant for Brew & Bean, a coffee shop in Bengaluru.
Answer using ONLY the numbered sources in the message.
- Prices are in rupees. Quote a price exactly as it appears in a source.
- If the sources do not answer the question, say you don't know and suggest asking at the counter.
- Be brief and friendly. Mention allergens if the customer asks about them.
- Cite sources like [1]."""


def price_check(answer, docs):
    """Return the rupee amounts in the answer that appear in NO retrieved source."""
    source_text = " ".join(d["text"] for d in docs)
    source_amounts = set(re.findall(r"Rs (\d+)", source_text))
    answer_amounts = re.findall(r"(?:Rs\.?|₹)\s?(\d+)", answer)
    return [a for a in answer_amounts if a not in source_amounts]


def ask(question, category=None):
    docs = search(question, category=category)
    context = "\n".join(f"[{n}] {d['text']}" for n, d in enumerate(docs, 1))
    resp = client.models.generate_content(
        model=GEN_MODEL,
        contents=f"Sources:\n{context}\n\nCustomer question: {question}",
        config=types.GenerateContentConfig(system_instruction=SYSTEM, temperature=0.2),
    )
    answer = resp.text
    return {"answer": answer, "sources": [d["id"] for d in docs],
            "unverified_prices": price_check(answer, docs)}


if __name__ == "__main__":
    while True:
        print(ask(input("\nYou: ")))

python assistant.py

from fastapi import FastAPI
from pydantic import BaseModel

from assistant import ask

app = FastAPI(title="Brew & Bean assistant")


class Question(BaseModel):
    question: str
    category: str | None = None  # optional filter: coffee, tea, bakery, food


@app.post("/ask")
def ask_endpoint(body: Question):
    return ask(body.question, body.category)

uvicorn api:app --reload

curl -X POST http://127.0.0.1:8000/ask -H "Content-Type: application/json" -d "{\"question\": \"What bakery items have nuts?\", \"category\": \"bakery\"}"

from search import search

# (question, id (or set of ids) of which at least one MUST appear in the top 3)
CASES = [
    ("How much is a cappuccino?", "menu:cappuccino"),
    ("Do you deliver?", "faq:1"),
    ("What time do you close?", "faq:0"),
    ("Something cold and not bitter", "menu:cold-brew"),
    ("Is there anything without gluten or milk?", {"menu:espresso", "menu:cold-brew"}),
    ("Can I get oat milk?", "faq:3"),
    ("Which items have nuts?", "menu:banana-bread"),
    ("Can I cancel my order?", "faq:2"),
]

hits = 0
for question, expected in CASES:
    ids = [d["id"] for d in search(question, k=3)]
    ok = bool(set(ids) & (expected if isinstance(expected, set) else {expected}))
    hits += ok
    print("PASS" if ok else "FAIL", "|", question, "->", ids)
print(f"\nRecall@3: {hits}/{len(CASES)}")

python eval_retrieval.py

My RAG retrieval test fails for this question: [question]. Expected document: [id] with text: [text]. It returned: [ids and texts]. I use BM25 plus embedding search merged with reciprocal rank fusion. Explain the most likely reason, and suggest two changes to the document text or the search code. Then tell me how to confirm each change helped.