Retrieval-Augmented Generation (RAG): Build, Evaluate and Improve a Q&A Assistant

import re, numpy as np, pandas as pd
DOCS = {
"returns.md": """# Returns policy
## Return window
Most products can be returned within 10 days of delivery. Mobiles and laptops have a 7-day replacement-only window for manufacturing defects.
## Conditions
Products must be unused, with original box, accessories and invoice. Opened earbuds and personal-care items cannot be returned for hygiene reasons.
## Refund timeline
Refunds are issued to the original payment method within 5–7 working days after the returned product passes quality check. UPI refunds usually arrive within 48 hours.""",
"warranty.md": """# Warranty
## Standard warranty
All electronics carry the manufacturer's warranty: 1 year for mobiles, laptops and smartwatches, 6 months for chargers and cables.
## Extended warranty
NovaMart Care adds 1 or 2 extra years and covers accidental liquid damage once per year. It must be bought within 30 days of delivery.
## How to claim
Book a claim in the app under Orders → Help. A technician visit is scheduled within 72 hours in metro cities and 5 days elsewhere.""",
"delivery.md": """# Delivery
## Delivery times
Metro cities: 1–2 days. Other cities: 3–5 days. Remote pin codes: up to 8 days.
## Delivery charges
Delivery is free on orders above ₹499. Below that a ₹49 fee applies. Same-day delivery costs ₹99 in Bengaluru, Delhi NCR, Mumbai and Pune.
## Damaged parcels
If the box is damaged on arrival, refuse the delivery or report it within 48 hours with photos for a free replacement.""",
"payments.md": """# Payments
## Methods
We accept UPI, credit and debit cards, net banking, NovaMart wallet and cash on delivery up to ₹10,000.
## No-cost EMI
No-cost EMI is available on orders above ₹3,000 with selected bank cards for 3, 6 or 9 months. The interest is given back as an instant discount.
## Failed payments
If money is debited but the order fails, it is automatically refunded within 3 working days.""",
"exchange.md": """# Exchange offers
## Old device exchange
Customers can exchange an old phone or laptop for an instant discount. The final value is confirmed by the delivery agent after a physical check.
## Eligibility
The device must switch on, have no cracked screen and match the declared model. Exchange is not available with cash on delivery.""",
"loyalty.md": """# NovaMart Plus
## Benefits
Plus members get free same-day delivery, early access to sales and 5% back as NovaCoins on every order.
## Cost
Membership costs ₹399 per year or ₹49 per month and can be cancelled any time; unused months are not refunded.""",
}
print(len(DOCS), "documents,", sum(len(t.split()) for t in DOCS.values()), "words")

def chunk_markdown(name, text, max_words=80):
    title = re.search(r"^# (.+)$", text, re.M).group(1); chunks = []
    for sec in re.split(r"^## ", text, flags=re.M)[1:]:
        heading, body = sec.split("\n", 1); words = body.split()
        for i in range(0, len(words), max_words - 15):                 # 15-word overlap if a section is long
            chunks.append({"id": f"{name}#{heading.strip().lower().replace(' ', '-')}" + (f"-{i}" if i else ""), "doc": name, "title": title, "section": heading.strip(),
                           "text": f"{title} > {heading.strip()}: " + " ".join(words[i:i + max_words])})
    return chunks
chunks = pd.DataFrame([c for n, t in DOCS.items() for c in chunk_markdown(n, t)])
print(len(chunks), "chunks"); print(chunks[["id", "text"]].head(4).to_string(index=False, max_colwidth=90))

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
word_vec = TfidfVectorizer(ngram_range=(1, 2), sublinear_tf=True, stop_words="english").fit(chunks.text)
char_vec = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), sublinear_tf=True).fit(chunks.text)     # robust to typos and word forms
Wd, Cd = word_vec.transform(chunks.text), char_vec.transform(chunks.text)
def retrieve(question, k=3, doc_filter=None):
    s = 0.6 * cosine_similarity(word_vec.transform([question]), Wd)[0] + 0.4 * cosine_similarity(char_vec.transform([question]), Cd)[0]
    if doc_filter: s = np.where(chunks.doc.isin(doc_filter), s, -1)
    top = np.argsort(-s)[:k]
    return chunks.iloc[top].assign(score=s[top].round(3))
print(retrieve("How long does a refund take if I paid by UPI?")[["id", "score"]].to_string(index=False))
print(retrieve("is there emi without interest")[["id", "score"]].to_string(index=False))

SYSTEM = """You are NovaMart's customer-support assistant.
Answer ONLY from the documents provided. If they do not contain the answer, say "I don't have that information" and suggest contacting support.
Cite the document id in square brackets after each fact, like [returns.md#refund-timeline]. Keep answers under 80 words."""
def build_prompt(question, k=3):
    ctx = retrieve(question, k)
    docs = "\n".join(f'<document id="{r.id}">\n{r.text}\n</document>' for r in ctx.itertuples())
    return f"<documents>\n{docs}\n</documents>\n\n<question>{question}</question>", ctx
prompt, ctx = build_prompt("My earbuds box was damaged when it arrived. What should I do?")
print(prompt)

import anthropic
client = anthropic.Anthropic()
response = client.messages.create(
    model="claude-opus-5-5", max_tokens=2000, system=SYSTEM,
    output_config={"effort": "low"},                     # a short grounded answer does not need deep reasoning
    messages=[{"role": "user", "content": prompt}],
)
answer = next(b.text for b in response.content if b.type == "text")

# Or let the API produce verifiable citations: pass chunks as document blocks with citations enabled
content = [{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": r.text}, "title": r.id, "citations": {"enabled": True}} for r in ctx.itertuples()]
content.append({"type": "text", "text": question})
response = client.messages.create(model="claude-opus-5-5", max_tokens=2000, messages=[{"role": "user", "content": content}])
for block in response.content:                            # text blocks carry .citations with the exact cited text
    print(block.text, [c.cited_text for c in (getattr(block, "citations", None) or [])])

def answer_offline(question, k=3, min_score=0.12):
    ctx = retrieve(question, k)
    if ctx.score.max() < min_score: return "I don't have that information. Please contact NovaMart support.", ctx
    sents = [(s.strip(), r.id) for r in ctx.itertuples() for s in re.split(r"(?<=[.!?])\s+", r.text.split(": ", 1)[1]) if s.strip()]
    sv = TfidfVectorizer(ngram_range=(1, 2), stop_words="english").fit([s for s, _ in sents] + [question])
    sc = cosine_similarity(sv.transform([question]), sv.transform([s for s, _ in sents]))[0]
    best = [sents[i] for i in np.argsort(-sc)[:2] if sc[i] > 0]
    return " ".join(f"{s} [{cid}]" for s, cid in best) or "I don't have that information.", ctx
for q in ["My earbuds box was damaged when it arrived. What should I do?", "How long does a refund take if I paid by UPI?",
          "Can I use cash on delivery with an exchange offer?", "Do you deliver to Sri Lanka?"]:
    a, _ = answer_offline(q); print(f"Q: {q}\nA: {a}\n")

TEST = [
    ("How many days do I have to return a speaker?", "returns.md#return-window", "10 days"),
    ("Can I return opened earbuds?", "returns.md#conditions", "cannot be returned"),
    ("When will my UPI refund arrive?", "returns.md#refund-timeline", "48 hours"),
    ("How long is the warranty on a charger?", "warranty.md#standard-warranty", "6 months"),
    ("Does NovaMart Care cover water damage?", "warranty.md#extended-warranty", "liquid damage"),
    ("What is the delivery fee for a ₹300 order?", "delivery.md#delivery-charges", "₹49"),
    ("Is same-day delivery available in Pune?", "delivery.md#delivery-charges", "Pune"),
    ("Is no cost emi available on a 2500 rupee order?", "payments.md#no-cost-emi", "above ₹3,000"),
    ("Money was debited but my order failed", "payments.md#failed-payments", "3 working days"),
    ("Can I exchange a phone with a cracked screen?", "exchange.md#eligibility", "no cracked screen"),
    ("How much does Plus membership cost?", "loyalty.md#cost", "₹399"),
    ("What are the benefits of NovaMart Plus?", "loyalty.md#benefits", "same-day"),
]
rows = []
for q, gold, fact in TEST:
    ret = retrieve(q, k=3).id.tolist(); ans, _ = answer_offline(q)
    rows.append({"question": q[:42], "hit@1": gold == ret[0], "hit@3": gold in ret, "rank": (ret.index(gold) + 1) if gold in ret else None,
                 "fact_in_answer": fact.lower() in ans.lower(), "cites_gold": gold in ans})
ev = pd.DataFrame(rows); print(ev.to_string(index=False))
mrr = np.mean([1 / r if r else 0 for r in ev["rank"]])
print(f"\nretrieval: hit@1 {ev['hit@1'].mean():.0%} | hit@3 {ev['hit@3'].mean():.0%} | MRR {mrr:.2f} || answers: key fact present {ev.fact_in_answer.mean():.0%} | cites the right section {ev.cites_gold.mean():.0%}")