Course materials · AI Applications

L20. Text Classification

English translation

This notebook uses a small review dataset to examine the basic sequence classification workflow.

Key objectives:

The Appendix covers external datasets, Hugging Face login, and running pretrained models.

1. Toy review dataset

Fix label meanings as follows

review_texts = [
    "great battery bright screen",
    "bad camera slow app",
    "screen size okay",
    "love fast performance",
    "broken charger bad battery",
    "average design normal speed",
    "great sound love display",
    "slow response hate keyboard",
    "okay price average camera",
    "bright screen fast app",
    "bad screen okay battery",
    "normal package average manual",
]

review_labels = [2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1]
label_names = {0: "negative", 1: "neutral", 2: "positive"}

print("sample count:", len(review_texts))
print("label count:", len(review_labels))
for text, label in zip(review_texts[:3], review_labels[:3]):
    print(label, label_names[label], "|", text)
sample count: 12
label count: 12
2 positive | great battery bright screen
0 negative | bad camera slow app
1 neutral | screen size okay

2. Tokenization

The first step splits text into a word token list. Here, the text is already lowercase and uses simple whitespace separation.

tokenized_texts = [text.split() for text in review_texts]

print(tokenized_texts[0])
print("first length:", len(tokenized_texts[0]))
['great', 'battery', 'bright', 'screen']
first length: 4

3. Vocabulary indexing

A model does not compute directly on word strings. First build a vocabulary and convert each word to an integer index.

vocab = sorted({word for tokens in tokenized_texts for word in tokens})
word_to_idx = {"<pad>": 0}
for word in vocab:
    word_to_idx[word] = len(word_to_idx)
idx_to_word = {idx: word for word, idx in word_to_idx.items()}

indexed_texts = [[word_to_idx[word] for word in tokens] for tokens in tokenized_texts]

print("vocab size:", len(word_to_idx))
print("word_to_idx sample:", list(word_to_idx.items())[:8])
print("first indexed text:", indexed_texts[0])
vocab size: 28
word_to_idx sample: [('<pad>', 0), ('app', 1), ('average', 2), ('bad', 3), ('battery', 4), ('bright', 5), ('broken', 6), ('camera', 7)]
first indexed text: [12, 4, 5, 23]

4. Padding

Batching requires matching sequence lengths. Pad shorter sentences with <pad> index 0.

import numpy as np

max_len = max(len(seq) for seq in indexed_texts)
padded_sequences = np.zeros((len(indexed_texts), max_len), dtype=np.int64)

for i, seq in enumerate(indexed_texts):
    padded_sequences[i, :len(seq)] = seq

labels = np.array(review_labels, dtype=np.int64)

print("padded shape:", padded_sequences.shape)
print("first padded sequence:", padded_sequences[0].tolist())
print("first words:", [idx_to_word[int(i)] for i in padded_sequences[0]])
padded shape: (12, 4)
first padded sequence: [12, 4, 5, 23]
first words: ['great', 'battery', 'bright', 'screen']

5. Embedding

nn.Embedding(vocab_size, embedding_dim) converts word indices into dense vectors.

For input shape (batch, seq_len), the embedding output shape is (batch, seq_len, embedding_dim).

import torch
import torch.nn as nn
import torch.optim as optim

torch.manual_seed(7)

x = torch.tensor(padded_sequences, dtype=torch.long)
y = torch.tensor(labels, dtype=torch.long)

embedding = nn.Embedding(num_embeddings=len(word_to_idx), embedding_dim=5, padding_idx=0)
embedded = embedding(x)

print("x shape:", tuple(x.shape))
print("embedded shape:", tuple(embedded.shape))
x shape: (12, 4)
embedded shape: (12, 4, 5)

6. Sequence representation

Classification assigns one label to an entire sequence. A simple approach creates a sequence vector by averaging only the token embeddings that are not padding.

mask = (x != 0).unsqueeze(-1)
lengths = mask.sum(dim=1).clamp(min=1)
sequence_vectors = (embedded * mask).sum(dim=1) / lengths

print("mask shape:", tuple(mask.shape))
print("sequence_vectors shape:", tuple(sequence_vectors.shape))
mask shape: (12, 4, 1)
sequence_vectors shape: (12, 5)

7. Text classifier

Combine the Embedding and classifier into one model.

class MeanEmbeddingClassifier(nn.Module):
    def __init__(self, vocab_size, embedding_dim, class_count):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)
        self.classifier = nn.Linear(embedding_dim, class_count)

    def forward(self, token_ids):
        embedded = self.embedding(token_ids)
        mask = (token_ids != 0).unsqueeze(-1)
        lengths = mask.sum(dim=1).clamp(min=1)
        sequence_vectors = (embedded * mask).sum(dim=1) / lengths
        logits = self.classifier(sequence_vectors)
        return logits

model = MeanEmbeddingClassifier(
    vocab_size=len(word_to_idx),
    embedding_dim=8,
    class_count=3,
)

logits = model(x)
print("logits shape:", tuple(logits.shape))
logits shape: (12, 3)

8. Training

CrossEntropyLoss takes logits and class index targets.

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.05)

for epoch in range(201):
    optimizer.zero_grad()
    logits = model(x)
    loss = criterion(logits, y)
    loss.backward()
    optimizer.step()

    if epoch % 50 == 0:
        pred = logits.argmax(dim=1)
        acc = (pred == y).float().mean().item()
        print(f"epoch={epoch:3d} loss={loss.item():.4f} accuracy={acc:.2f}")
epoch=  0 loss=1.1429 accuracy=0.33
epoch= 50 loss=0.0001 accuracy=1.00
epoch=100 loss=0.0001 accuracy=1.00
epoch=150 loss=0.0001 accuracy=1.00
epoch=200 loss=0.0001 accuracy=1.00

9. Output verification

After training, inspect predicted labels and mismatches alongside the text, as well as accuracy.

with torch.no_grad():
    logits = model(x)
    probabilities = torch.softmax(logits, dim=1)
    pred = probabilities.argmax(dim=1)
    accuracy = (pred == y).float().mean().item()

print("probabilities shape:", tuple(probabilities.shape))
print("pred:", pred.tolist())
print("accuracy:", accuracy)

mismatch_indices = (pred != y).nonzero(as_tuple=True)[0].tolist()
if len(mismatch_indices) == 0:
    print("No mismatch in this toy example.")
else:
    for i in mismatch_indices:
        print("index:", i)
        print("text:", review_texts[i])
        print("true:", label_names[int(y[i])])
        print("pred:", label_names[int(pred[i])])
probabilities shape: (12, 3)
pred: [2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1]
accuracy: 1.0
No mismatch in this toy example.

10. Replay checklist