--- title: RNNPlus keywords: fastai sidebar: home_sidebar summary: "These are RNN, LSTM and GRU PyTorch implementations created by Ignacio Oguiza - timeseriesAI@gmail.com based on:" description: "These are RNN, LSTM and GRU PyTorch implementations created by Ignacio Oguiza - timeseriesAI@gmail.com based on:" nb_path: "nbs/105_models.RNNPlus.ipynb" ---
The idea of including a feature extractor to the RNN network comes from the solution developed by the UPSTAGE team (https://www.kaggle.com/songwonho, https://www.kaggle.com/limerobot and https://www.kaggle.com/jungikhyo). They finished in 3rd position in Kaggle's Google Brain - Ventilator Pressure Prediction competition. They used a Conv1d + Stacked LSTM architecture.
bs = 16
c_in = 3
seq_len = 12
c_out = 2
xb = torch.rand(bs, c_in, seq_len)
test_eq(RNNPlus(c_in, c_out)(xb).shape, [bs, c_out])
test_eq(RNNPlus(c_in, c_out, hidden_size=100, n_layers=2, bias=True, rnn_dropout=0.2, bidirectional=True, fc_dropout=0.5)(xb).shape, [bs, c_out])
test_eq(RNNPlus(c_in, c_out, hidden_size=[100, 50, 10], bias=True, rnn_dropout=0.2, bidirectional=True, fc_dropout=0.5)(xb).shape, [bs, c_out])
test_eq(RNNPlus(c_in, c_out, hidden_size=[100], n_layers=2, bias=True, rnn_dropout=0.2, bidirectional=True, fc_dropout=0.5)(xb).shape,
[bs, c_out])
test_eq(LSTMPlus(c_in, c_out, hidden_size=100, n_layers=2, bias=True, rnn_dropout=0.2, bidirectional=True, fc_dropout=0.5)(xb).shape, [bs, c_out])
test_eq(GRUPlus(c_in, c_out, hidden_size=100, n_layers=2, bias=True, rnn_dropout=0.2, bidirectional=True, fc_dropout=0.5)(xb).shape, [bs, c_out])
test_eq(RNNPlus(c_in, c_out, seq_len, last_step=False)(xb).shape, [bs, c_out])
test_eq(RNNPlus(c_in, c_out, seq_len, last_step=False)(xb).shape, [bs, c_out])
test_eq(RNNPlus(c_in, c_out, seq_len, hidden_size=100, n_layers=2, bias=True, rnn_dropout=0.2, bidirectional=True, fc_dropout=0.5, last_step=False)(xb).shape,
[bs, c_out])
test_eq(LSTMPlus(c_in, c_out, seq_len, last_step=False)(xb).shape, [bs, c_out])
test_eq(GRUPlus(c_in, c_out, seq_len, last_step=False)(xb).shape, [bs, c_out])
feature_extractor = MultiConv1d(c_in, kss=[1,3,5,7])
custom_head = nn.Sequential(Transpose(1,2), nn.Linear(8,8), nn.SELU(), nn.Linear(8, 1), Squeeze())
test_eq(LSTMPlus(c_in, c_out, seq_len, hidden_size=[32,16,8,4], bidirectional=True,
feature_extractor=feature_extractor, custom_head=custom_head)(xb).shape, [bs, seq_len])
feature_extractor = MultiConv1d(c_in, kss=[1,3,5,7], keep_original=True)
custom_head = nn.Sequential(Transpose(1,2), nn.Linear(8,8), nn.SELU(), nn.Linear(8, 1), Squeeze())
test_eq(LSTMPlus(c_in, c_out, seq_len, hidden_size=[32,16,8,4], bidirectional=True,
feature_extractor=feature_extractor, custom_head=custom_head)(xb).shape, [bs, seq_len])
bs = 16
c_in = 3
seq_len = 12
c_out = 2
x1 = torch.rand(bs,1,seq_len)
x2 = torch.randint(0,3,(bs,1,seq_len))
x3 = torch.randint(0,5,(bs,1,seq_len))
xb = torch.cat([x1,x2,x3],1)
custom_head = partial(create_mlp_head, fc_dropout=0.5)
test_eq(LSTMPlus(c_in, c_out, seq_len, last_step=False, custom_head=custom_head)(xb).shape, [bs, c_out])
custom_head = partial(create_pool_head, concat_pool=True, fc_dropout=0.5)
test_eq(LSTMPlus(c_in, c_out, seq_len, last_step=False, custom_head=custom_head)(xb).shape, [bs, c_out])
custom_head = partial(create_pool_plus_head, fc_dropout=0.5)
test_eq(LSTMPlus(c_in, c_out, seq_len, last_step=False, custom_head=custom_head)(xb).shape, [bs, c_out])
custom_head = partial(create_conv_head)
test_eq(LSTMPlus(c_in, c_out, seq_len, last_step=False, custom_head=custom_head)(xb).shape, [bs, c_out])
test_eq(LSTMPlus(c_in, c_out, seq_len, hidden_size=[100, 50], n_layers=2, bias=True, rnn_dropout=0.2, bidirectional=True)(xb).shape, [bs, c_out])
n_embeds = [3, 5]
cat_pos = [1, 2]
custom_head = partial(create_conv_head)
m = LSTMPlus(c_in, c_out, seq_len, hidden_size=[100, 50], n_layers=2, bias=True, rnn_dropout=0.2, bidirectional=True, n_embeds=n_embeds, cat_pos=cat_pos)
test_eq(m(xb).shape, [bs, c_out])
from tsai.data.all import *
from tsai.models.utils import *
dsid = 'NATOPS'
bs = 16
X, y, splits = get_UCR_data(dsid, return_split=False)
tfms = [None, [Categorize()]]
dls = get_ts_dls(X, y, tfms=tfms, splits=splits, bs=bs)
model = build_ts_model(LSTMPlus, dls=dls)
print(model[-1])
learn = Learner(dls, model, metrics=accuracy)
learn.fit_one_cycle(1, 3e-3)
model = LSTMPlus(dls.vars, dls.c, dls.len, last_step=False)
learn = Learner(dls, model, metrics=accuracy)
learn.fit_one_cycle(1, 3e-3)
custom_head = partial(create_pool_head, concat_pool=True)
model = LSTMPlus(dls.vars, dls.c, dls.len, last_step=False, custom_head=custom_head)
learn = Learner(dls, model, metrics=accuracy)
learn.fit_one_cycle(1, 3e-3)
custom_head = partial(create_pool_plus_head, concat_pool=True)
model = LSTMPlus(dls.vars, dls.c, dls.len, last_step=False, custom_head=custom_head)
learn = Learner(dls, model, metrics=accuracy)
learn.fit_one_cycle(1, 3e-3)
m = RNNPlus(c_in, c_out, seq_len, hidden_size=100,n_layers=2,bidirectional=True,rnn_dropout=.5,fc_dropout=.5)
print(m)
print(total_params(m))
m(xb).shape
m = LSTMPlus(c_in, c_out, seq_len, hidden_size=100,n_layers=2,bidirectional=True,rnn_dropout=.5,fc_dropout=.5)
print(m)
print(total_params(m))
m(xb).shape
m = GRUPlus(c_in, c_out, seq_len, hidden_size=100,n_layers=2,bidirectional=True,rnn_dropout=.5,fc_dropout=.5)
print(m)
print(total_params(m))
m(xb).shape