Customising Models: Prompting vs RAG vs Fine-Tuning, LoRA, Distillation and Open-Weight Models

import json, numpy as np, pandas as pd
rv = pd.read_csv("customer_reviews.csv")
SYSTEM = "Classify the review's sentiment as positive, negative or neutral. Reply with one word."
def to_example(text, label):
    return {"messages": [{"role": "system", "content": SYSTEM}, {"role": "user", "content": text}, {"role": "assistant", "content": label}]}
data = rv.sample(frac=1, random_state=0)
train, val, test = np.split(data, [int(.8 * len(data)), int(.9 * len(data))])
with open("train.jsonl", "w", encoding="utf-8") as f:
    for r in train.itertuples(): f.write(json.dumps(to_example(r.review_text, r.sentiment), ensure_ascii=False) + "\n")
print(len(train), "train |", len(val), "validation |", len(test), "test")
print(open("train.jsonl", encoding="utf-8").readline()[:220])
print("label balance:", train.sentiment.value_counts(normalize=True).round(2).to_dict())
import os; os.remove("train.jsonl")

def lora_params(d, k, r): return r * (d + k)
d = k = 4096                                                   # one attention projection in a ~7–8B model
for r in (4, 8, 16, 64):
    print(f"rank {r:2d}: {lora_params(d, k, r):,} trainable vs {d * k:,} full ({100 * lora_params(d, k, r) / (d * k):.2f}%)")
layers, mats = 32, 4                                           # 32 transformer layers × q, k, v, o projections
total_full = 8e9; total_lora = layers * mats * lora_params(4096, 4096, 16)
print(f"whole model: LoRA r=16 trains {total_lora / 1e6:.1f}M parameters ≈ {100 * total_lora / total_full:.2f}% of 8B")
print(f"memory for weights in 16-bit: {total_full * 2 / 1e9:.0f} GB | 4-bit (QLoRA): {total_full * 0.5 / 1e9:.0f} GB")

rng = np.random.default_rng(0)
true_update = rng.normal(size=(64, 4)) @ rng.normal(size=(4, 64))           # a genuinely low-rank change
U, S, Vt = np.linalg.svd(true_update + rng.normal(0, .01, (64, 64)))
for r in (1, 2, 4, 8):
    approx = (U[:, :r] * S[:r]) @ Vt[:r]
    print(f"rank-{r} approximation keeps {100 * (1 - np.linalg.norm(true_update - approx) / np.linalg.norm(true_update)):.1f}% of the update with {r * 128} numbers instead of 4,096")

# Typical open-model LoRA fine-tune (Hugging Face) - run on a GPU (e.g. Colab)
from datasets import load_dataset
from peft import LoraConfig
from trl import SFTTrainer, SFTConfig
ds = load_dataset("json", data_files={"train": "train.jsonl", "validation": "val.jsonl"})
trainer = SFTTrainer(
    model="Qwen/Qwen2.5-1.5B-Instruct",                   # small open model
    train_dataset=ds["train"], eval_dataset=ds["validation"],
    peft_config=LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05, target_modules="all-linear"),
    args=SFTConfig(output_dir="sentiment-lora", num_train_epochs=2, per_device_train_batch_size=8, learning_rate=2e-4),
)
trainer.train(); trainer.save_model()                    # saves only the small adapter

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.metrics import accuracy_score
teacher_acc = 0.93                                             # suppose a large LLM labels reviews with ~93% accuracy
noisy = train.copy(); flip = rng.random(len(noisy)) > teacher_acc
noisy.loc[flip, "sentiment"] = rng.choice(["positive", "negative", "neutral"], flip.sum())
for name, labels in [("human labels", train.sentiment), ("teacher (LLM) labels", noisy.sentiment)]:
    student = make_pipeline(TfidfVectorizer(ngram_range=(1, 2), min_df=2), LogisticRegression(max_iter=2000, C=5)).fit(train.review_text, labels)
    print(f"student trained on {name:22s} → test accuracy {accuracy_score(test.sentiment, student.predict(test.review_text)):.3f}")

def api_cost(req, tin, tout, pin, pout): return req * (tin * pin + tout * pout) / 1e6
monthly_requests = 3_000_000; tin, tout = 400, 60
api_small = api_cost(monthly_requests, tin, tout, 0.10, 0.50)        # small hosted model (Claude Haiku 5.5 list price, USD/M tokens)
api_large = api_cost(monthly_requests, tin, tout, 4.00, 20.00)       # frontier model
gpu_month = 1.2 * 24 * 30                                             # one cloud GPU at ~$1.2/hour, always on
engineering = 2500                                                    # share of an engineer's time to run it (USD/month, illustrative)
print(f"API small model: ${api_small:,.0f}/month | API frontier model: ${api_large:,.0f}/month | self-host 1 GPU: ${gpu_month + engineering:,.0f}/month (incl. ops)")