Recurrent Neural Network

Imports

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim

# for reproducibility
torch.manual_seed(100)

<torch._C.Generator at 0x79192b2df150>

1 Dataset Construction

# Dictionary
sample_sentence = 'hi!hello.'
char_set = list(set(sample_sentence))
dic = {c: i for i, c in enumerate(char_set)}

# Parameters
dic_size = len(dic)
input_size = dic_size
hidden_size = dic_size

# Dataset setting
x_batch = []
y_batch = []

x_data = [dic[c] for c in sample_sentence[:-1]]
x_one_hot = [np.eye(dic_size)[x] for x in x_data]
y_data = [dic[c] for c in sample_sentence[1:]]

x_batch.append(x_one_hot)
y_batch.append(y_data)

# To torch tensors
X = torch.FloatTensor(x_batch)
Y = torch.LongTensor(y_batch)
print(X.shape)
print(Y.shape)

torch.Size([1, 8, 7])
torch.Size([1, 8])

2 RNN model

• Input (input structure)
  • Input type: torch.Tensor
  • Input shape: (N x S x E)
    • N: Batch size, S: Sequence length, E: Embedding size
    • Only when batch_first=True
  • Input ‘hi!hello’
    • (1, 8, 7)
• Hidden (output structure)
  • Hidden type: torch.Tensor
  • Output ‘i!hello.’
    • (1, 8, 7)

# Model
learning_rate = 0.1
training_epochs = 50
model = nn.RNN(input_size, hidden_size, batch_first=True)

# define cost/loss & optimizer
criterion = nn.CrossEntropyLoss()    # Softmax
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

# train
for epoch in range(training_epochs):
  optimizer.zero_grad()
  outputs, _status = model(X)
  loss = criterion(outputs.reshape(-1, dic_size), Y.reshape(-1))
  loss.backward()
  optimizer.step()
  if epoch % 5 == 4:
    result = outputs.data.numpy().argmax(axis=2)
    result_str = ''.join([char_set[c] for c in np.squeeze(result)])
    print('epoch: ',epoch, 'loss: ', loss.item(), 'prediction: ', result_str, 'true Y: ', sample_sentence[1:])

epoch:  4 loss:  1.2792741060256958 prediction:  i!hillo. true Y:  i!hello.
epoch:  9 loss:  0.9466097354888916 prediction:  i!hillo. true Y:  i!hello.
epoch:  14 loss:  0.8066588640213013 prediction:  i!hillo. true Y:  i!hello.
epoch:  19 loss:  0.7434771656990051 prediction:  i!hillo. true Y:  i!hello.
epoch:  24 loss:  0.712984561920166 prediction:  i!hillo. true Y:  i!hello.
epoch:  29 loss:  0.6942877769470215 prediction:  i!hillo. true Y:  i!hello.
epoch:  34 loss:  0.6845752596855164 prediction:  i!hillo. true Y:  i!hello.
epoch:  39 loss:  0.6790880560874939 prediction:  i!hillo. true Y:  i!hello.
epoch:  44 loss:  0.6758877635002136 prediction:  i!hillo. true Y:  i!hello.
epoch:  49 loss:  0.6733384132385254 prediction:  i!hillo. true Y:  i!hello.

3 Assignment

Train an RNN using the following 3 sentences as batch data (refer to the incomplete code below, the exercise code above, and the execution results)

• ‘howareyou’
• ‘whats up?’
• ‘iamgreat.’

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim

# for reproducibility
torch.manual_seed(100)

# Dictionary
sample_sentences = ['howareyou', 'whats up?', 'iamgreat.']
char_set = list(set(''.join(sample_sentences)))
dic = {c: i for i, c in enumerate(char_set)}

# Parameters
dic_size = len(dic)
input_size = dic_size
hidden_size = dic_size

# Dataset setting
input_batch = []
target_batch = []

for sentence in sample_sentences:


# To torch tensors
X = torch.FloatTensor(input_batch)
Y = torch.LongTensor(target_batch)

# Model
learning_rate = 0.05
training_epochs = 500
model = nn.RNN(input_size, hidden_size, batch_first=True)

# define cost/loss & optimizer
criterion = nn.CrossEntropyLoss()    # Softmax
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

# train
for epoch in range(training_epochs):

result = outputs.data.numpy().argmax(axis=2)
for sentence in result:
  print(''.join([char_set[c] for c in np.squeeze(sentence)]))