NLP & Transformers — From Tokenization to Fine-Tuning

Raw text → Normalize → Tokenize → Vectorize → Model → Decode

king - man + woman ≈ queen    (famous word analogy)
paris - france + italy ≈ rome (geographic analogy)

# Scaled dot-product attention (the core operation in every transformer)
def attention(Q, K, V, mask=None):
    d_k = Q.shape[-1]
    scores = Q @ K.transpose(-2, -1) / math.sqrt(d_k)  # Scale by sqrt(d_k)
    if mask is not None:
        scores = scores.masked_fill(mask == 0, -1e9)
    weights = torch.softmax(scores, dim=-1)             # Probability distribution
    return weights @ V, weights                         # Weighted sum of values

Input tokens → Token Embeddings + Positional Encoding
    ↓
[Multi-Head Self-Attention → Add & Norm → Feed-Forward → Add & Norm] × N
    ↓
Output (contextualized representations)

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
text = "The quick brown fox jumped over the lazy dogs."

tokens = tokenizer(text, return_tensors="pt")
print(tokenizer.convert_ids_to_tokens(tokens.input_ids[0]))
# ['[CLS]', 'the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'lazy', 'dogs', '.', '[SEP]']

# Subword splitting for unknown/rare words
rare_text = "Pneumonoultramicroscopicsilicovolcanoconiosis is challenging."
rare_tokens = tokenizer.tokenize(rare_text)
print(rare_tokens)
# ['pneumon', '##oul', '##tra', '##micro', '##scop', '##ics', '##ili', ...]

from datasets import load_dataset
from transformers import (
    AutoTokenizer,
    AutoModelForSequenceClassification,
    TrainingArguments,
    Trainer,
)
import numpy as np
from sklearn.metrics import accuracy_score, f1_score

# Load dataset
dataset = load_dataset("imdb")

# Tokenizer
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")

def tokenize(batch):
    return tokenizer(
        batch["text"],
        truncation=True,
        max_length=512,
        padding="max_length",
    )

tokenized = dataset.map(tokenize, batched=True, batch_size=1000)
tokenized.set_format("torch", columns=["input_ids", "attention_mask", "label"])

# Model: DistilBERT is 40% smaller than BERT, 60% faster, retains 97% accuracy
model = AutoModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased",
    num_labels=2
)

# Metrics
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    return {
        "accuracy": accuracy_score(labels, preds),
        "f1": f1_score(labels, preds, average="binary"),
    }

# Training arguments
args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    warmup_ratio=0.1,
    weight_decay=0.01,
    learning_rate=2e-5,
    evaluation_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    fp16=True,       # Mixed precision for speed
)

trainer = Trainer(
    model=model,
    args=args,
    train_dataset=tokenized["train"].select(range(5000)),   # Subset for demo
    eval_dataset=tokenized["test"].select(range(1000)),
    compute_metrics=compute_metrics,
)

trainer.train()
# Fine-tuned BERT reaches ~93% accuracy on IMDB vs ~90% for TF-IDF + LR

from sentence_transformers import SentenceTransformer
import numpy as np

model = SentenceTransformer("all-MiniLM-L6-v2")   # 80MB, fast, excellent quality

docs = [
    "Python is a high-level programming language",
    "Machine learning requires lots of data",
    "Neural networks are inspired by the brain",
    "The stock market crashed yesterday",
    "Interest rates affect mortgage payments",
]

doc_embeddings = model.encode(docs, normalize_embeddings=True)

query = "deep learning with neural nets"
query_embedding = model.encode(query, normalize_embeddings=True)

# Cosine similarity is just dot product when vectors are normalized
similarities = doc_embeddings @ query_embedding

print("Semantic similarity to:", query)
for doc, sim in sorted(zip(docs, similarities), key=lambda x: -x[1]):
    print(f"  {sim:.4f}: {doc}")
# Output correctly ranks neural networks > ML > Python > finance docs

from groq import Groq

client = Groq()

# ── System prompt pattern ─────────────────────────────────────────────────────
response = client.chat.completions.create(
    model="llama-3.3-70b-versatile",
    messages=[
        {
            "role": "system",
            "content": "You are a sentiment analysis expert. Analyze text and return JSON only."
        },
        {
            "role": "user",
            "content": "Analyze: 'The film had stunning visuals but a weak plot.'"
        }
    ],
    response_format={"type": "json_object"},
)

# ── Few-shot pattern ──────────────────────────────────────────────────────────
few_shot = [
    {"role": "user",      "content": "Classify: 'The model crashed.' → "},
    {"role": "assistant", "content": "TECHNICAL_ISSUE"},
    {"role": "user",      "content": "Classify: 'Payment declined.' → "},
    {"role": "assistant", "content": "BILLING_ISSUE"},
    {"role": "user",      "content": "Classify: 'Cannot login to account.' → "},
]

# ── Chain-of-thought ──────────────────────────────────────────────────────────
cot_prompt = """
<task>Classify the customer support ticket below.</task>
<instructions>
1. Identify the core problem described
2. Consider which department handles this type of issue
3. Rate urgency from 1-5
4. Output JSON: {"category": "...", "urgency": N, "summary": "..."}
</instructions>
<ticket>My API key stopped working and I have a demo in 2 hours!</ticket>
"""