--- title: Layers keywords: fastai sidebar: home_sidebar summary: "Helper function used to build PyTorch timeseries models." description: "Helper function used to build PyTorch timeseries models." nb_path: "nbs/100_models.layers.ipynb" ---
{% raw %}
{% endraw %} {% raw %}
{% endraw %} {% raw %}

noop[source]

noop(x=None, *args, **kwargs)

Do nothing

{% endraw %} {% raw %}
{% endraw %} {% raw %}

init_lin_zero[source]

init_lin_zero(m)

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class SwishBeta[source]

SwishBeta(beta=1.0) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class Chomp1d[source]

Chomp1d(chomp_size) :: Module

Base class for all neural network modules.

Your models should also subclass this class.

Modules can also contain other Modules, allowing to nest them in a tree structure. You can assign the submodules as regular attributes::

import torch.nn as nn
import torch.nn.functional as F

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5)
        self.conv2 = nn.Conv2d(20, 20, 5)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        return F.relu(self.conv2(x))

Submodules assigned in this way will be registered, and will have their parameters converted too when you call :meth:to, etc.

:ivar training: Boolean represents whether this module is in training or evaluation mode. :vartype training: bool

{% endraw %} {% raw %}
{% endraw %} {% raw %}

same_padding1d[source]

same_padding1d(seq_len, ks, stride=1, dilation=1)

Same padding formula as used in Tensorflow

{% endraw %} {% raw %}

class Pad1d[source]

Pad1d(padding, value=0.0) :: ConstantPad1d

Pads the input tensor boundaries with a constant value.

For N-dimensional padding, use :func:torch.nn.functional.pad().

Args: padding (int, tuple): the size of the padding. If is int, uses the same padding in both boundaries. If a 2-tuple, uses (:math:\text{padding\_left}, :math:\text{padding\_right})

Shape:

- Input: :math:`(C, W_{in})` or :math:`(N, C, W_{in})`.
- Output: :math:`(C, W_{out})` or :math:`(N, C, W_{out})`, where

  :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}`

Examples::

>>> m = nn.ConstantPad1d(2, 3.5)
>>> input = torch.randn(1, 2, 4)
>>> input
tensor([[[-1.0491, -0.7152, -0.0749,  0.8530],
         [-1.3287,  1.8966,  0.1466, -0.2771]]])
>>> m(input)
tensor([[[ 3.5000,  3.5000, -1.0491, -0.7152, -0.0749,  0.8530,  3.5000,
           3.5000],
         [ 3.5000,  3.5000, -1.3287,  1.8966,  0.1466, -0.2771,  3.5000,
           3.5000]]])
>>> m = nn.ConstantPad1d(2, 3.5)
>>> input = torch.randn(1, 2, 3)
>>> input
tensor([[[ 1.6616,  1.4523, -1.1255],
         [-3.6372,  0.1182, -1.8652]]])
>>> m(input)
tensor([[[ 3.5000,  3.5000,  1.6616,  1.4523, -1.1255,  3.5000,  3.5000],
         [ 3.5000,  3.5000, -3.6372,  0.1182, -1.8652,  3.5000,  3.5000]]])
>>> # using different paddings for different sides
>>> m = nn.ConstantPad1d((3, 1), 3.5)
>>> m(input)
tensor([[[ 3.5000,  3.5000,  3.5000,  1.6616,  1.4523, -1.1255,  3.5000],
         [ 3.5000,  3.5000,  3.5000, -3.6372,  0.1182, -1.8652,  3.5000]]])
{% endraw %} {% raw %}

class SameConv1d[source]

SameConv1d(ni, nf, ks=3, stride=1, dilation=1, padding:Union[str, Tuple[~T]][int]]=0, groups:int=1, bias:bool=True, padding_mode:str='zeros', device=None, dtype=None) :: Module

Conv1d with padding='same'

{% endraw %} {% raw %}
{% endraw %} {% raw %}

same_padding2d[source]

same_padding2d(H, W, ks, stride=(1, 1), dilation=(1, 1))

Same padding formula as used in Tensorflow

{% endraw %} {% raw %}

class Pad2d[source]

Pad2d(padding, value=0.0) :: ConstantPad2d

Pads the input tensor boundaries with a constant value.

For N-dimensional padding, use :func:torch.nn.functional.pad().

Args: padding (int, tuple): the size of the padding. If is int, uses the same padding in all boundaries. If a 4-tuple, uses (:math:\text{padding\_left}, :math:\text{padding\_right}, :math:\text{padding\_top}, :math:\text{padding\_bottom})

Shape:

- Input: :math:`(N, C, H_{in}, W_{in})` or :math:`(C, H_{in}, W_{in})`.
- Output: :math:`(N, C, H_{out}, W_{out})` or :math:`(C, H_{out}, W_{out})`, where

  :math:`H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}`

  :math:`W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}`

Examples::

>>> m = nn.ConstantPad2d(2, 3.5)
>>> input = torch.randn(1, 2, 2)
>>> input
tensor([[[ 1.6585,  0.4320],
         [-0.8701, -0.4649]]])
>>> m(input)
tensor([[[ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000,  3.5000],
         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000,  3.5000],
         [ 3.5000,  3.5000,  1.6585,  0.4320,  3.5000,  3.5000],
         [ 3.5000,  3.5000, -0.8701, -0.4649,  3.5000,  3.5000],
         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000,  3.5000],
         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000,  3.5000]]])
>>> # using different paddings for different sides
>>> m = nn.ConstantPad2d((3, 0, 2, 1), 3.5)
>>> m(input)
tensor([[[ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000],
         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000],
         [ 3.5000,  3.5000,  3.5000,  1.6585,  0.4320],
         [ 3.5000,  3.5000,  3.5000, -0.8701, -0.4649],
         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000]]])
{% endraw %} {% raw %}

class Conv2dSame[source]

Conv2dSame(ni, nf, ks=(3, 3), stride=(1, 1), dilation=(1, 1), padding:Union[str, Tuple[~T, ~T]][int]]=0, groups:int=1, bias:bool=True, padding_mode:str='zeros', device=None, dtype=None) :: Module

Conv2d with padding='same'

{% endraw %} {% raw %}

Conv2d[source]

Conv2d(ni, nf, kernel_size=None, ks=None, stride=1, padding='same', dilation=1, init='auto', bias_std=0.01, groups:int=1, bias:bool=True, padding_mode:str='zeros', device=None, dtype=None)

conv1d layer with padding='same', 'valid', or any integer (defaults to 'same')

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 2
c_in = 3
c_out = 5
h = 16
w = 20
t = torch.rand(bs, c_in, h, w)
test_eq(Conv2dSame(c_in, c_out, ks=3, stride=1, dilation=1, bias=False)(t).shape, (bs, c_out, h, w))
test_eq(Conv2dSame(c_in, c_out, ks=(3, 1), stride=1, dilation=1, bias=False)(t).shape, (bs, c_out, h, w))
test_eq(Conv2dSame(c_in, c_out, ks=3, stride=(1, 1), dilation=(2, 2), bias=False)(t).shape, (bs, c_out, h, w))
test_eq(Conv2dSame(c_in, c_out, ks=3, stride=(2, 2), dilation=(1, 1), bias=False)(t).shape, (bs, c_out, h//2, w//2))
test_eq(Conv2dSame(c_in, c_out, ks=3, stride=(2, 2), dilation=(2, 2), bias=False)(t).shape, (bs, c_out, h//2, w//2))
test_eq(Conv2d(c_in, c_out, ks=3, padding='same', stride=1, dilation=1, bias=False)(t).shape, (bs, c_out, h, w))
{% endraw %} {% raw %}

class CausalConv1d[source]

CausalConv1d(ni, nf, ks, stride=1, dilation=1, groups=1, bias=True) :: Conv1d

Applies a 1D convolution over an input signal composed of several input planes.

In the simplest case, the output value of the layer with input size :math:(N, C_{\text{in}}, L) and output :math:(N, C_{\text{out}}, L_{\text{out}}) can be precisely described as:

.. math:: \text{out}(Ni, C{\text{out}j}) = \text{bias}(C{\text{out}j}) + \sum{k = 0}^{C{in} - 1} \text{weight}(C{\text{out}_j}, k) \star \text{input}(N_i, k)

where :math:\star is the valid cross-correlation_ operator, :math:N is a batch size, :math:C denotes a number of channels, :math:L is a length of signal sequence.

This module supports :ref:TensorFloat32<tf32_on_ampere>.

  • :attr:stride controls the stride for the cross-correlation, a single number or a one-element tuple.

  • :attr:padding controls the amount of padding applied to the input. It can be either a string {'valid', 'same'} or a tuple of ints giving the amount of implicit padding applied on both sides.

  • :attr:dilation controls the spacing between the kernel points; also known as the à trous algorithm. It is harder to describe, but this link_ has a nice visualization of what :attr:dilation does.

  • :attr:groups controls the connections between inputs and outputs. :attr:in_channels and :attr:out_channels must both be divisible by :attr:groups. For example,

    • At groups=1, all inputs are convolved to all outputs.
    • At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels and producing half the output channels, and both subsequently concatenated.
    • At groups= :attr:in_channels, each input channel is convolved with its own set of filters (of size :math:\frac{\text{out\_channels}}{\text{in\_channels}}).

Note: When groups == in_channels and out_channels == K * in_channels, where K is a positive integer, this operation is also known as a "depthwise convolution".

In other words, for an input of size :math:`(N, C_{in}, L_{in})`,
a depthwise convolution with a depthwise multiplier `K` can be performed with the arguments
:math:`(C_\text{in}=C_\text{in}, C_\text{out}=C_\text{in} \times \text{K}, ..., \text{groups}=C_\text{in})`.

Note: In some circumstances when given tensors on a CUDA device and using CuDNN, this operator may select a nondeterministic algorithm to increase performance. If this is undesirable, you can try to make the operation deterministic (potentially at a performance cost) by setting torch.backends.cudnn.deterministic = True. See :doc:/notes/randomness for more information.

Note: padding='valid' is the same as no padding. padding='same' pads the input so the output has the shape as the input. However, this mode doesn't support any stride values other than 1.

Args: in_channels (int): Number of channels in the input image out_channels (int): Number of channels produced by the convolution kernel_size (int or tuple): Size of the convolving kernel stride (int or tuple, optional): Stride of the convolution. Default: 1 padding (int, tuple or str, optional): Padding added to both sides of the input. Default: 0 padding_mode (string, optional): 'zeros', 'reflect', 'replicate' or 'circular'. Default: 'zeros' dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 bias (bool, optional): If True, adds a learnable bias to the output. Default: True

Shape:

- Input: :math:`(N, C_{in}, L_{in})`
- Output: :math:`(N, C_{out}, L_{out})` where

  .. math::
      L_{out} = \left\lfloor\frac{L_{in} + 2 \times \text{padding} - \text{dilation}
                \times (\text{kernel\_size} - 1) - 1}{\text{stride}} + 1\right\rfloor

Attributes: weight (Tensor): the learnable weights of the module of shape :math:(\text{out\_channels}, \frac{\text{in\_channels}}{\text{groups}}, \text{kernel\_size}). The values of these weights are sampled from :math:\mathcal{U}(-\sqrt{k}, \sqrt{k}) where :math:k = \frac{groups}{C_\text{in} * \text{kernel\_size}} bias (Tensor): the learnable bias of the module of shape (outchannels). If :attr:bias is True, then the values of these weights are sampled from :math:\mathcal{U}(-\sqrt{k}, \sqrt{k}) where :math:`k = \frac{groups}{C\text{in} * \text{kernel_size}}`

Examples::

>>> m = nn.Conv1d(16, 33, 3, stride=2)
>>> input = torch.randn(20, 16, 50)
>>> output = m(input)

.. _cross-correlation: https://en.wikipedia.org/wiki/Cross-correlation

.. _link: https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md

{% endraw %} {% raw %}
{% endraw %} {% raw %}

Conv1d[source]

Conv1d(ni, nf, kernel_size=None, ks=None, stride=1, padding='same', dilation=1, init='auto', bias_std=0.01, groups:int=1, bias:bool=True, padding_mode:str='zeros', device=None, dtype=None)

conv1d layer with padding='same', 'causal', 'valid', or any integer (defaults to 'same')

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 2
c_in = 3
c_out = 5
seq_len = 512
t = torch.rand(bs, c_in, seq_len)
dilation = 1
test_eq(CausalConv1d(c_in, c_out, ks=3, dilation=dilation)(t).shape, Conv1d(c_in, c_out, ks=3, padding="same", dilation=dilation)(t).shape)
dilation = 2
test_eq(CausalConv1d(c_in, c_out, ks=3, dilation=dilation)(t).shape, Conv1d(c_in, c_out, ks=3, padding="same", dilation=dilation)(t).shape)
{% endraw %} {% raw %}
bs = 2
ni = 3
nf = 5
seq_len = 6
ks = 3
t = torch.rand(bs, c_in, seq_len)
test_eq(Conv1d(ni, nf, ks, padding=0)(t).shape, (bs, c_out, seq_len - (2 * (ks//2))))
test_eq(Conv1d(ni, nf, ks, padding='valid')(t).shape, (bs, c_out, seq_len - (2 * (ks//2))))
test_eq(Conv1d(ni, nf, ks, padding='same')(t).shape, (bs, c_out, seq_len))
test_eq(Conv1d(ni, nf, ks, padding='causal')(t).shape, (bs, c_out, seq_len))
test_error('use kernel_size or ks but not both simultaneously', Conv1d, ni, nf, kernel_size=3, ks=3)
test_error('you need to pass a ks', Conv1d, ni, nf)
---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-13-c128bf6f8b89> in <module>
      9 test_eq(Conv1d(ni, nf, ks, padding='same')(t).shape, (bs, c_out, seq_len))
     10 test_eq(Conv1d(ni, nf, ks, padding='causal')(t).shape, (bs, c_out, seq_len))
---> 11 test_error('use kernel_size or ks but not both simultaneously', Conv1d, ni, nf, kernel_size=3, ks=3)
     12 test_error('you need to pass a ks', Conv1d, ni, nf)

NameError: name 'test_error' is not defined
{% endraw %} {% raw %}
conv = Conv1d(ni, nf, ks, padding='same')
init_linear(conv, None, init='auto', bias_std=.01)
conv
{% endraw %} {% raw %}
conv = Conv1d(ni, nf, ks, padding='causal')
init_linear(conv, None, init='auto', bias_std=.01)
conv
{% endraw %} {% raw %}
conv = Conv1d(ni, nf, ks, padding='valid')
init_linear(conv, None, init='auto', bias_std=.01)
weight_norm(conv)
conv
{% endraw %} {% raw %}
conv = Conv1d(ni, nf, ks, padding=0)
init_linear(conv, None, init='auto', bias_std=.01)
weight_norm(conv)
conv
{% endraw %} {% raw %}

class SeparableConv1d[source]

SeparableConv1d(ni, nf, ks, stride=1, padding='same', dilation=1, bias=True, bias_std=0.01) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 64
c_in = 6
c_out = 5
seq_len = 512
t = torch.rand(bs, c_in, seq_len)
test_eq(SeparableConv1d(c_in, c_out, 3)(t).shape, (bs, c_out, seq_len))
{% endraw %} {% raw %}

class AddCoords1d[source]

AddCoords1d() :: Module

Add coordinates to ease position identification without modifying mean and std

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 2
c_in = 3
c_out = 5
seq_len = 50

t = torch.rand(bs, c_in, seq_len)
t = (t - t.mean()) / t.std()
test_eq(AddCoords1d()(t).shape, (bs, c_in + 1, seq_len))
new_t = AddCoords1d()(t)
test_close(new_t.mean(),0, 1e-2)
test_close(new_t.std(), 1, 1e-2)
{% endraw %} {% raw %}

class ConvBlock[source]

ConvBlock(ni, nf, kernel_size=None, ks=3, stride=1, padding='same', bias=None, bias_std=0.01, norm='Batch', zero_norm=False, bn_1st=True, act=ReLU, act_kwargs={}, init='auto', dropout=0.0, xtra=None, coord=False, separable=False, **kwargs) :: Sequential

Create a sequence of conv1d (ni to nf), activation (if act_cls) and norm_type layers.

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class ResBlock1dPlus[source]

ResBlock1dPlus(expansion, ni, nf, coord=False, stride=1, groups=1, reduction=None, nh1=None, nh2=None, dw=False, g2=1, sa=False, sym=False, norm='Batch', zero_norm=True, act_cls=ReLU, ks=3, pool=AvgPool, pool_first=True, padding=None, bias=None, ndim=2, norm_type=<NormType.Batch: 1>, bn_1st=True, transpose=False, init='auto', xtra=None, bias_std=0.01, dilation:Tuple[~T, ~T]][int]=1, padding_mode:str='zeros', device=None, dtype=None) :: Module

Resnet block from ni to nh with stride

{% endraw %} {% raw %}
{% endraw %} {% raw %}

SEModule1d[source]

SEModule1d(ni, reduction=16, act=ReLU, act_kwargs={})

Squeeze and excitation module for 1d

{% endraw %} {% raw %}
{% endraw %} {% raw %}
t = torch.rand(8, 32, 12)
test_eq(SEModule1d(t.shape[1], 16, act=nn.ReLU, act_kwargs={})(t).shape, t.shape)
{% endraw %} {% raw %}

Norm[source]

Norm(nf, ndim=1, norm='Batch', zero_norm=False, init=True, **kwargs)

Norm layer with nf features and ndim with auto init.

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 2
ni = 3
nf = 5
sl = 4
ks = 5

t = torch.rand(bs, ni, sl)
test_eq(ConvBlock(ni, nf, ks)(t).shape, (bs, nf, sl))
test_eq(ConvBlock(ni, nf, ks, padding='causal')(t).shape, (bs, nf, sl))
test_eq(ConvBlock(ni, nf, ks, coord=True)(t).shape, (bs, nf, sl))
{% endraw %} {% raw %}
test_eq(BN1d(ni)(t).shape, (bs, ni, sl))
test_eq(BN1d(ni).weight.data.mean().item(), 1.)
test_eq(BN1d(ni, zero_norm=True).weight.data.mean().item(), 0.)
{% endraw %} {% raw %}
test_eq(ConvBlock(ni, nf, ks, norm='batch', zero_norm=True)[1].weight.data.unique().item(), 0)
test_ne(ConvBlock(ni, nf, ks, norm='batch', zero_norm=False)[1].weight.data.unique().item(), 0)
test_eq(ConvBlock(ni, nf, ks, bias=False)[0].bias, None)
ConvBlock(ni, nf, ks, act=Swish, coord=True)
{% endraw %} {% raw %}

class LinLnDrop[source]

LinLnDrop(n_in, n_out, ln=True, p=0.0, act=None, lin_first=False) :: Sequential

Module grouping LayerNorm1d, Dropout and Linear layers

{% endraw %} {% raw %}
{% endraw %} {% raw %}
LinLnDrop(2, 3, p=.5)
{% endraw %} {% raw %}

class LambdaPlus[source]

LambdaPlus(func, *args, **kwargs) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class Squeeze[source]

Squeeze(dim=-1) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class Unsqueeze[source]

Unsqueeze(dim=-1) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class Add[source]

Add() :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class Concat[source]

Concat(dim=1) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class Permute[source]

Permute(*dims) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class Transpose[source]

Transpose(*dims, contiguous=False) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class View[source]

View(*size) :: Module

Reshape x to size

{% endraw %} {% raw %}

class Reshape[source]

Reshape(*shape) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class Max[source]

Max(dim=None, keepdim=False) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class LastStep[source]

LastStep() :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class SoftMax[source]

SoftMax(dim=-1) :: Module

SoftMax layer

{% endraw %} {% raw %}

class Clamp[source]

Clamp(min=None, max=None) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class Clip[source]

Clip(min=None, max=None) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class ReZero[source]

ReZero(module) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 2
nf = 5
sl = 4

t = torch.rand(bs, nf, sl)
test_eq(Permute(0,2,1)(t).shape, (bs, sl, nf))
test_eq(Max(1)(t).shape, (bs, sl))
test_eq(Transpose(1,2)(t).shape, (bs, sl, nf))
test_eq(Transpose(1,2, contiguous=True)(t).shape, (bs, sl, nf))
test_eq(View(-1, 2, 10)(t).shape, (bs, 1, 2, 10))
test_eq(Reshape(-1, 2, 10)(t).shape, (bs, 1, 2, 10))
Transpose(1,2), Permute(0,2,1), View(-1, 2, 10), Transpose(1,2, contiguous=True), Reshape(-1, 2, 10), Noop
{% endraw %} {% raw %}

class DropPath[source]

DropPath(p=None) :: Module

Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

It's similar to Dropout but it drops individual connections instead of nodes. Original code in https://github.com/rwightman/pytorch-image-models (timm library)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
t = torch.ones(100,2,3)
test_eq(DropPath(0.)(t), t)
assert DropPath(0.5)(t).max() >= 1
{% endraw %} {% raw %}

class Sharpen[source]

Sharpen(T=0.5) :: Module

This is used to increase confidence in predictions - MixMatch paper

{% endraw %} {% raw %}
{% endraw %} {% raw %}
n_samples = 1000
n_classes = 3

t = (torch.rand(n_samples, n_classes) - .5) * 10
probas = F.softmax(t, -1)
sharpened_probas = Sharpen()(probas)
plt.plot(probas.flatten().sort().values, color='r')
plt.plot(sharpened_probas.flatten().sort().values, color='b')
plt.show()
test_gt(sharpened_probas[n_samples//2:].max(-1).values.sum().item(), probas[n_samples//2:].max(-1).values.sum().item())
{% endraw %} {% raw %}

class Sequential[source]

Sequential(*args) :: Sequential

Class that allows you to pass one or multiple inputs

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class TimeDistributed[source]

TimeDistributed(module, low_mem=False, tdim=1) :: Module

Applies module over tdim identically for each step, use low_mem to compute one at a time.

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class Temp_Scale[source]

Temp_Scale(temp=1.0, dirichlet=False) :: Module

Used to perform Temperature Scaling (dirichlet=False) or Single-parameter Dirichlet calibration (dirichlet=True)

{% endraw %} {% raw %}

class Vector_Scale[source]

Vector_Scale(n_classes=1, dirichlet=False) :: Module

Used to perform Vector Scaling (dirichlet=False) or Diagonal Dirichlet calibration (dirichlet=True)

{% endraw %} {% raw %}

class Matrix_Scale[source]

Matrix_Scale(n_classes=1, dirichlet=False) :: Module

Used to perform Matrix Scaling (dirichlet=False) or Dirichlet calibration (dirichlet=True)

{% endraw %} {% raw %}

get_calibrator[source]

get_calibrator(calibrator=None, n_classes=1, **kwargs)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 2
c_out = 3

t = torch.rand(bs, c_out)
for calibrator, cal_name in zip(['temp', 'vector', 'matrix'], ['Temp_Scale', 'Vector_Scale', 'Matrix_Scale']): 
    cal = get_calibrator(calibrator, n_classes=c_out)
#     print(calibrator)
#     print(cal.weight, cal.bias, '\n')
    test_eq(cal(t), t)
    test_eq(cal.__class__.__name__, cal_name)
for calibrator, cal_name in zip(['dtemp', 'dvector', 'dmatrix'], ['Temp_Scale', 'Vector_Scale', 'Matrix_Scale']):
    cal = get_calibrator(calibrator, n_classes=c_out)
#     print(calibrator)
#     print(cal.weight, cal.bias, '\n')
    test_eq(cal(t), F.log_softmax(t, dim=1))
    test_eq(cal.__class__.__name__, cal_name)
{% endraw %} {% raw %}
bs = 2
c_out = 3

t = torch.rand(bs, c_out)

test_eq(Temp_Scale()(t).shape, t.shape)
test_eq(Vector_Scale(c_out)(t).shape, t.shape)
test_eq(Matrix_Scale(c_out)(t).shape, t.shape)
test_eq(Temp_Scale(dirichlet=True)(t).shape, t.shape)
test_eq(Vector_Scale(c_out, dirichlet=True)(t).shape, t.shape)
test_eq(Matrix_Scale(c_out, dirichlet=True)(t).shape, t.shape)

test_eq(Temp_Scale()(t), t)
test_eq(Vector_Scale(c_out)(t), t)
test_eq(Matrix_Scale(c_out)(t), t)
{% endraw %} {% raw %}
bs = 2
c_out = 5

t = torch.rand(bs, c_out)
test_eq(Vector_Scale(c_out)(t), t)
test_eq(Vector_Scale(c_out).weight.data, torch.ones(c_out))
test_eq(Vector_Scale(c_out).weight.requires_grad, True)
test_eq(type(Vector_Scale(c_out).weight), torch.nn.parameter.Parameter)
{% endraw %} {% raw %}
bs = 2
c_out = 3
weight = 2
bias = 1

t = torch.rand(bs, c_out)
test_eq(Matrix_Scale(c_out)(t).shape, t.shape)
test_eq(Matrix_Scale(c_out).weight.requires_grad, True)
test_eq(type(Matrix_Scale(c_out).weight), torch.nn.parameter.Parameter)
{% endraw %} {% raw %}

class LogitAdjustmentLayer[source]

LogitAdjustmentLayer(class_priors) :: Module

Logit Adjustment for imbalanced datasets

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs, n_classes = 16, 3
class_priors = torch.rand(n_classes)
logits = torch.randn(bs, n_classes) * 2
test_eq(LogitAdjLayer(class_priors)(logits), logits + class_priors)
{% endraw %} {% raw %}

class PPV[source]

PPV(dim=-1) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class PPAuc[source]

PPAuc(dim=-1) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class MaxPPVPool1d[source]

MaxPPVPool1d() :: Module

Drop-in replacement for AdaptiveConcatPool1d - multiplies nf by 2

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 2
nf = 5
sl = 4

t = torch.rand(bs, nf, sl)
test_eq(MaxPPVPool1d()(t).shape, (bs, nf*2, 1))
test_eq(MaxPPVPool1d()(t).shape, AdaptiveConcatPool1d(1)(t).shape)
{% endraw %} {% raw %}

class AdaptiveWeightedAvgPool1d[source]

AdaptiveWeightedAvgPool1d(n_in, seq_len, mult=2, n_layers=2, ln=False, dropout=0.5, act=ReLU(), zero_init=True) :: Module

Global Pooling layer that performs a weighted average along the temporal axis

It can be considered as a channel-wise form of local temporal attention. Inspired by the paper: Hyun, J., Seong, H., & Kim, E. (2019). Universal Pooling--A New Pooling Method for Convolutional Neural Networks. arXiv preprint arXiv:1907.11440.

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class GAP1d[source]

GAP1d(output_size=1) :: Module

Global Adaptive Pooling + Flatten

{% endraw %} {% raw %}

class GACP1d[source]

GACP1d(output_size=1) :: Module

Global AdaptiveConcatPool + Flatten

{% endraw %} {% raw %}

class GAWP1d[source]

GAWP1d(n_in, seq_len, n_layers=2, ln=False, dropout=0.5, act=ReLU(), zero_init=False) :: Module

Global AdaptiveWeightedAvgPool1d + Flatten

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class GlobalWeightedAveragePool1d[source]

GlobalWeightedAveragePool1d(n_in, seq_len) :: Module

Global Weighted Average Pooling layer

Inspired by Building Efficient CNN Architecture for Offline Handwritten Chinese Character Recognition https://arxiv.org/pdf/1804.01259.pdf

{% endraw %} {% raw %}

gwa_pool_head[source]

gwa_pool_head(n_in, c_out, seq_len, bn=True, fc_dropout=0.0)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
t = torch.randn(16, 64, 50)
head = gwa_pool_head(64, 5, 50)
test_eq(head(t).shape, (16, 5))
{% endraw %} {% raw %}

class AttentionalPool1d[source]

AttentionalPool1d(n_in, c_out, bn=False) :: Module

Global Adaptive Pooling layer inspired by Attentional Pooling for Action Recognition https://arxiv.org/abs/1711.01467

{% endraw %} {% raw %}

class GAttP1d[source]

GAttP1d(n_in, c_out, bn=False) :: Sequential

A sequential container. Modules will be added to it in the order they are passed in the constructor. Alternatively, an OrderedDict of modules can be passed in. The forward() method of Sequential accepts any input and forwards it to the first module it contains. It then "chains" outputs to inputs sequentially for each subsequent module, finally returning the output of the last module.

The value a Sequential provides over manually calling a sequence of modules is that it allows treating the whole container as a single module, such that performing a transformation on the Sequential applies to each of the modules it stores (which are each a registered submodule of the Sequential).

What's the difference between a Sequential and a :class:torch.nn.ModuleList? A ModuleList is exactly what it sounds like--a list for storing Module s! On the other hand, the layers in a Sequential are connected in a cascading way.

Example::

# Using Sequential to create a small model. When `model` is run,
# input will first be passed to `Conv2d(1,20,5)`. The output of
# `Conv2d(1,20,5)` will be used as the input to the first
# `ReLU`; the output of the first `ReLU` will become the input
# for `Conv2d(20,64,5)`. Finally, the output of
# `Conv2d(20,64,5)` will be used as input to the second `ReLU`
model = nn.Sequential(
          nn.Conv2d(1,20,5),
          nn.ReLU(),
          nn.Conv2d(20,64,5),
          nn.ReLU()
        )

# Using Sequential with OrderedDict. This is functionally the
# same as the above code
model = nn.Sequential(OrderedDict([
          ('conv1', nn.Conv2d(1,20,5)),
          ('relu1', nn.ReLU()),
          ('conv2', nn.Conv2d(20,64,5)),
          ('relu2', nn.ReLU())
        ]))
{% endraw %} {% raw %}

attentional_pool_head[source]

attentional_pool_head(n_in, c_out, seq_len=None, bn=True, **kwargs)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs, c_in, seq_len = 16, 1, 50
c_out = 3
t = torch.rand(bs, c_in, seq_len)
test_eq(GAP1d()(t).shape, (bs, c_in))
test_eq(GACP1d()(t).shape, (bs, c_in*2))
bs, c_in, seq_len = 16, 4, 50
t = torch.rand(bs, c_in, seq_len)
test_eq(GAP1d()(t).shape, (bs, c_in))
test_eq(GACP1d()(t).shape, (bs, c_in*2))
test_eq(GAWP1d(c_in, seq_len, n_layers=2, ln=False, dropout=0.5, act=nn.ReLU(), zero_init=False)(t).shape, (bs, c_in))
test_eq(GAWP1d(c_in, seq_len, n_layers=2, ln=False, dropout=0.5, act=nn.ReLU(), zero_init=False)(t).shape, (bs, c_in))
test_eq(GAWP1d(c_in, seq_len, n_layers=1, ln=False, dropout=0.5, zero_init=False)(t).shape, (bs, c_in))
test_eq(GAWP1d(c_in, seq_len, n_layers=1, ln=False, dropout=0.5, zero_init=True)(t).shape, (bs, c_in))
test_eq(AttentionalPool1d(c_in, c_out)(t).shape, (bs, c_out, 1))
{% endraw %} {% raw %}
bs, c_in, seq_len = 16, 128, 50
c_out = 14
t = torch.rand(bs, c_in, seq_len)
attp = attentional_pool_head(c_in, c_out)
test_eq(attp(t).shape, (bs, c_out))
{% endraw %} {% raw %}

class GEGLU[source]

GEGLU() :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}

class ReGLU[source]

ReGLU() :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}

get_act_fn[source]

get_act_fn(act, **act_kwargs)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
test_eq(get_act_fn(nn.ReLU).__repr__(), "ReLU()")
test_eq(get_act_fn(nn.ReLU()).__repr__(), "ReLU()")
test_eq(get_act_fn(nn.LeakyReLU, negative_slope=0.05).__repr__(), "LeakyReLU(negative_slope=0.05)")
test_eq(get_act_fn('reglu').__repr__(), "ReGLU()")
test_eq(get_act_fn('leakyrelu', negative_slope=0.05).__repr__(), "LeakyReLU(negative_slope=0.05)")
{% endraw %} {% raw %}

create_pool_head[source]

create_pool_head(n_in, c_out, seq_len=None, concat_pool=False, fc_dropout=0.0, bn=False, y_range=None, **kwargs)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 12
c_out = 2
seq_len = 20
t = torch.rand(bs, nf, seq_len)
test_eq(create_pool_head(nf, c_out, seq_len, fc_dropout=0.5)(t).shape, (bs, c_out))
test_eq(create_pool_head(nf, c_out, seq_len, concat_pool=True, fc_dropout=0.5)(t).shape, (bs, c_out))
create_pool_head(nf, c_out, seq_len, concat_pool=True, bn=True, fc_dropout=.5)
{% endraw %} {% raw %}

max_pool_head[source]

max_pool_head(n_in, c_out, seq_len, fc_dropout=0.0, bn=False, y_range=None, **kwargs)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 12
c_out = 2
seq_len = 20
t = torch.rand(bs, nf, seq_len)
test_eq(max_pool_head(nf, c_out, seq_len, fc_dropout=0.5)(t).shape, (bs, c_out))
{% endraw %} {% raw %}

create_pool_plus_head[source]

create_pool_plus_head(*args, lin_ftrs=None, fc_dropout=0.0, concat_pool=True, bn_final=False, lin_first=False, y_range=None)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 12
c_out = 2
seq_len = 20
t = torch.rand(bs, nf, seq_len)
test_eq(create_pool_plus_head(nf, c_out, seq_len, fc_dropout=0.5)(t).shape, (bs, c_out))
test_eq(create_pool_plus_head(nf, c_out, concat_pool=True, fc_dropout=0.5)(t).shape, (bs, c_out))
create_pool_plus_head(nf, c_out, seq_len, fc_dropout=0.5)
{% endraw %} {% raw %}

create_conv_head[source]

create_conv_head(*args, adaptive_size=None, y_range=None)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 12
c_out = 2
seq_len = 20
t = torch.rand(bs, nf, seq_len)
test_eq(create_conv_head(nf, c_out, seq_len)(t).shape, (bs, c_out))
test_eq(create_conv_head(nf, c_out, adaptive_size=50)(t).shape, (bs, c_out))
create_conv_head(nf, c_out, 50)
{% endraw %} {% raw %}

create_mlp_head[source]

create_mlp_head(nf, c_out, seq_len=None, flatten=True, fc_dropout=0.0, bn=False, lin_first=False, y_range=None)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 12
c_out = 2
seq_len = 20
t = torch.rand(bs, nf, seq_len)
test_eq(create_mlp_head(nf, c_out, seq_len, fc_dropout=0.5)(t).shape, (bs, c_out))
t = torch.rand(bs, nf, seq_len)
create_mlp_head(nf, c_out, seq_len, bn=True, fc_dropout=.5)
{% endraw %} {% raw %}

create_fc_head[source]

create_fc_head(nf, c_out, seq_len=None, flatten=True, lin_ftrs=None, y_range=None, fc_dropout=0.0, bn=False, bn_final=False, act=ReLU(inplace=True))

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 12
c_out = 2
seq_len = 20
t = torch.rand(bs, nf, seq_len)
test_eq(create_fc_head(nf, c_out, seq_len, fc_dropout=0.5)(t).shape, (bs, c_out))
create_mlp_head(nf, c_out, seq_len, bn=True, fc_dropout=.5)
{% endraw %} {% raw %}

create_rnn_head[source]

create_rnn_head(*args, fc_dropout=0.0, bn=False, y_range=None)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 12
c_out = 2
seq_len = 20
t = torch.rand(bs, nf, seq_len)
test_eq(create_rnn_head(nf, c_out, seq_len, fc_dropout=0.5)(t).shape, (bs, c_out))
create_rnn_head(nf, c_out, seq_len, bn=True, fc_dropout=.5)
{% endraw %} {% raw %}

imputation_head[source]

imputation_head(c_in, c_out, seq_len=None, ks=1, y_range=None, fc_dropout=0.0)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 12
ni = 2
seq_len = 20
t = torch.rand(bs, nf, seq_len)
head = imputation_head(nf, ni, seq_len=None, ks=1, y_range=None, fc_dropout=0.)
test_eq(head(t).shape, (bs, ni, seq_len))
head = imputation_head(nf, ni, seq_len=None, ks=1, y_range=(.3,.7), fc_dropout=0.)
test_ge(head(t).min(), .3)
test_le(head(t).max(), .7)
y_range = (tensor([0.1000, 0.1000, 0.1000, 0.1000, 0.2000, 0.2000, 0.2000, 0.2000, 0.3000,
                   0.3000, 0.3000, 0.3000]),
           tensor([0.6000, 0.6000, 0.6000, 0.6000, 0.7000, 0.7000, 0.7000, 0.7000, 0.8000,
                   0.8000, 0.8000, 0.8000]))
test_ge(head(t).min(), .1)
test_le(head(t).max(), .9)
head = imputation_head(nf, ni, seq_len=None, ks=1, y_range=y_range, fc_dropout=0.)
head
{% endraw %} {% raw %}

class create_conv_lin_nd_head[source]

create_conv_lin_nd_head(n_in, n_out, seq_len, d, conv_first=True, conv_bn=False, lin_bn=False, fc_dropout=0.0, **kwargs) :: Sequential

Module to create a nd output head

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 5
seq_len = 10
d = 2
targ = torch.randint(0, c, (bs,d))
t = torch.randn(bs, nf, seq_len)
head = conv_lin_nd_head(nf, c, seq_len, d, conv_first=True, fc_dropout=.5)
inp = head(t)
test_eq(inp.shape, (bs, d, c))
loss = CrossEntropyLossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 5
seq_len = 10
d = [2, 8]
targ = torch.randint(0, c, [bs]+d)
t = torch.randn(bs, nf, seq_len)
head = conv_lin_nd_head(nf, c, seq_len, d, conv_first=False, fc_dropout=.5)
inp = head(t)
test_eq(inp.shape, [bs]+d+[c])
loss = CrossEntropyLossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 1
seq_len = 10
d = 2
targ = torch.rand(bs, d)
t = torch.randn(bs, nf, seq_len)
head = conv_lin_nd_head(nf, c, seq_len, d, conv_first=False, fc_dropout=.5)
inp = head(t)
test_eq(inp.shape, (bs, d))
loss = L1LossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 1
seq_len = 10
d = [2,3]
targ = torch.rand(bs, *d)
t = torch.randn(bs, nf, seq_len)
head = conv_lin_nd_head(nf, c, seq_len, d, conv_first=False, fc_dropout=.5)
inp = head(t)
test_eq(inp.shape, [bs]+d)
loss = L1LossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}

class create_lin_nd_head[source]

create_lin_nd_head(n_in, n_out, seq_len, d, use_bn=False, fc_dropout=0.0) :: Sequential

Module to create a nd output head with linear layers

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 5
seq_len = 10
d = 2
targ = torch.randint(0, c, (bs,d))
t = torch.randn(bs, nf, seq_len)
head = lin_nd_head(nf, c, seq_len, d, fc_dropout=.5)
inp = head(t)
test_eq(inp.shape, (bs, d, c))
loss = CrossEntropyLossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 5
seq_len = 10
d = [2, 8]
targ = torch.randint(0, c, [bs]+d)
t = torch.randn(bs, nf, seq_len)
head = lin_nd_head(nf, c, seq_len, d, fc_dropout=.5)
inp = head(t)
test_eq(inp.shape, [bs]+d+[c])
loss = CrossEntropyLossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 1
seq_len = 10
d = 2
targ = torch.rand(bs, d)
t = torch.randn(bs, nf, seq_len)
head = lin_nd_head(nf, c, seq_len, d, fc_dropout=.5)
inp = head(t)
test_eq(inp.shape, (bs, d))
loss = L1LossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 1
seq_len = 10
d = [2,3]
targ = torch.rand(bs, *d)
t = torch.randn(bs, nf, seq_len)
head = lin_nd_head(nf, c, seq_len, d, fc_dropout=.5)
inp = head(t)
test_eq(inp.shape, [bs]+d)
loss = L1LossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}

class create_conv_3d_head[source]

create_conv_3d_head(n_in, n_out, seq_len, d, use_bn=False, **kwargs) :: Sequential

Module to create a nd output head with a convolutional layer

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 5
seq_len = 10
d = 10
targ = torch.randint(0, c, (bs,d))
t = torch.randn(bs, nf, seq_len)
head = conv_3d_head(nf, c, seq_len, d)
inp = head(t)
test_eq(inp.shape, (bs, d, c))
loss = CrossEntropyLossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}
bs = 16
nf = 32
c = 1
seq_len = 10
d = 10
targ = torch.rand(bs, d)
t = torch.randn(bs, nf, seq_len)
head = conv_3d_head(nf, c, seq_len, d)
inp = head(t)
test_eq(inp.shape, (bs, d))
loss = L1LossFlat()(inp, targ)
loss, head
{% endraw %} {% raw %}

universal_pool_head[source]

universal_pool_head(n_in, c_out, seq_len, mult=2, pool_n_layers=2, pool_ln=True, pool_dropout=0.5, pool_act=ReLU(), zero_init=True, bn=True, fc_dropout=0.0)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs, c_in, seq_len = 16, 128, 50
c_out = 14
t = torch.rand(bs, c_in, seq_len)
uph = universal_pool_head(c_in, c_out, seq_len)
test_eq(uph(t).shape, (bs, c_out))
uph = universal_pool_head(c_in, c_out, seq_len, 2)
test_eq(uph(t).shape, (bs, c_out))
{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs, c_in, seq_len = 16, 128, 50
c_out = 14
d = 5
t = torch.rand(bs, c_in, seq_len)
for head in heads: 
    print(head.__name__)
    if head.__name__ == "create_conv_3d_head":
        h = head(c_in, c_out, seq_len, seq_len)
        test_eq(h(t).shape, (bs, seq_len, c_out))
    elif 'nd' in head.__name__: 
        h = head(c_in, c_out, seq_len, d)
        test_eq(h(t).shape, (bs, d, c_out))
    else: 
        h = head(c_in, c_out, seq_len)
        test_eq(h(t).shape, (bs, c_out))
{% endraw %} {% raw %}

class SqueezeExciteBlock[source]

SqueezeExciteBlock(ni, reduction=16) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
bs = 2
ni = 32
sl = 4
t = torch.rand(bs, ni, sl)
test_eq(SqueezeExciteBlock(ni)(t).shape, (bs, ni, sl))
{% endraw %} {% raw %}

class GaussianNoise[source]

GaussianNoise(sigma=0.1, is_relative_detach=True) :: Module

Gaussian noise regularizer.

Args: sigma (float, optional): relative standard deviation used to generate the noise. Relative means that it will be multiplied by the magnitude of the value your are adding the noise to. This means that sigma can be the same regardless of the scale of the vector. is_relative_detach (bool, optional): whether to detach the variable before computing the scale of the noise. If False then the scale of the noise won't be seen as a constant but something to optimize: this will bias the network to generate vectors with smaller values.

{% endraw %} {% raw %}
{% endraw %} {% raw %}
t = torch.ones(2,3,4)
test_ne(GaussianNoise()(t), t)
test_eq(GaussianNoise()(t).shape, t.shape)
t = torch.ones(2,3)
test_ne(GaussianNoise()(t), t)
test_eq(GaussianNoise()(t).shape, t.shape)
t = torch.ones(2)
test_ne(GaussianNoise()(t), t)
test_eq(GaussianNoise()(t).shape, t.shape)
{% endraw %} {% raw %}

gambler_loss[source]

gambler_loss(reward=2)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
model_output = torch.rand(16, 3)
targets = torch.randint(0, 2, (16,))
criterion = gambler_loss(2)
criterion(model_output, targets)
{% endraw %} {% raw %}

CrossEntropyLossOneHot[source]

CrossEntropyLossOneHot(output, target, **kwargs)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
output = torch.rand(16, 2)
target = torch.randint(0, 2, (16,))
CrossEntropyLossOneHot(output, target)
{% endraw %} {% raw %}
from tsai.data.transforms import OneHot
output = nn.Parameter(torch.rand(16, 2))
target = torch.randint(0, 2, (16,))
one_hot_target = OneHot()(target)
CrossEntropyLossOneHot(output, one_hot_target)
{% endraw %} {% raw %}
ttest_tensor(a, b)
{% endraw %} {% raw %}

ttest_bin_loss[source]

ttest_bin_loss(output, target)

{% endraw %} {% raw %}

ttest_reg_loss[source]

ttest_reg_loss(output, target)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
for _ in range(100):
    output = torch.rand(256, 2)
    target = torch.randint(0, 2, (256,))
    test_close(ttest_bin_loss(output, target).item(), 
               ttest_ind(nn.Softmax(dim=-1)(output[:, 1])[target == 0], nn.Softmax(dim=-1)(output[:, 1])[target == 1], equal_var=False)[0], eps=1e-3)
{% endraw %} {% raw %}

class CenterLoss[source]

CenterLoss(c_out, logits_dim=None) :: Module

Code in Pytorch has been slightly modified from: https://github.com/KaiyangZhou/pytorch-center-loss/blob/master/center_loss.py Based on paper: Wen et al. A Discriminative Feature Learning Approach for Deep Face Recognition. ECCV 2016.

Args: c_out (int): number of classes. logits_dim (int): dim 1 of the logits. By default same as c_out (for one hot encoded logits)

{% endraw %} {% raw %}

class CenterPlusLoss[source]

CenterPlusLoss(loss, c_out, λ=0.01, logits_dim=None) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
c_in = 10
x = torch.rand(64, c_in).to(device=default_device())
x = F.softmax(x, dim=1)
label = x.max(dim=1).indices
CenterLoss(c_in).to(x.device)(x, label), CenterPlusLoss(LabelSmoothingCrossEntropyFlat(), c_in).to(x.device)(x, label)
{% endraw %} {% raw %}
CenterPlusLoss(LabelSmoothingCrossEntropyFlat(), c_in)
{% endraw %} {% raw %}

class FocalLoss[source]

FocalLoss(gamma:float=2.0, weight=None, reduction:str='mean') :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
c_in = 10
x = torch.rand(64, c_in).to(device=default_device())
x = F.softmax(x, dim=1)
label = x.max(dim=1).indices
FocalLoss(c_in).to(x.device)(x, label)
{% endraw %} {% raw %}

class TweedieLoss[source]

TweedieLoss(p=1.5, eps=1e-10) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
c_in = 10
output = torch.rand(64).to(device=default_device())
target = torch.rand(64).to(device=default_device())
TweedieLoss().to(output.device)(output, target)
{% endraw %} {% raw %}

class PositionwiseFeedForward[source]

PositionwiseFeedForward(dim, dropout=0.0, act='reglu', mlp_ratio=1) :: Sequential

A sequential container. Modules will be added to it in the order they are passed in the constructor. Alternatively, an OrderedDict of modules can be passed in. The forward() method of Sequential accepts any input and forwards it to the first module it contains. It then "chains" outputs to inputs sequentially for each subsequent module, finally returning the output of the last module.

The value a Sequential provides over manually calling a sequence of modules is that it allows treating the whole container as a single module, such that performing a transformation on the Sequential applies to each of the modules it stores (which are each a registered submodule of the Sequential).

What's the difference between a Sequential and a :class:torch.nn.ModuleList? A ModuleList is exactly what it sounds like--a list for storing Module s! On the other hand, the layers in a Sequential are connected in a cascading way.

Example::

# Using Sequential to create a small model. When `model` is run,
# input will first be passed to `Conv2d(1,20,5)`. The output of
# `Conv2d(1,20,5)` will be used as the input to the first
# `ReLU`; the output of the first `ReLU` will become the input
# for `Conv2d(20,64,5)`. Finally, the output of
# `Conv2d(20,64,5)` will be used as input to the second `ReLU`
model = nn.Sequential(
          nn.Conv2d(1,20,5),
          nn.ReLU(),
          nn.Conv2d(20,64,5),
          nn.ReLU()
        )

# Using Sequential with OrderedDict. This is functionally the
# same as the above code
model = nn.Sequential(OrderedDict([
          ('conv1', nn.Conv2d(1,20,5)),
          ('relu1', nn.ReLU()),
          ('conv2', nn.Conv2d(20,64,5)),
          ('relu2', nn.ReLU())
        ]))
{% endraw %} {% raw %}

class TokenLayer[source]

TokenLayer(token=True) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
t = torch.randn(2,3,10)
m = PositionwiseFeedForward(10, dropout=0., act='reglu', mlp_ratio=1)
test_eq(m(t).shape, t.shape)
{% endraw %} {% raw %}

class ScaledDotProductAttention[source]

ScaledDotProductAttention(attn_dropout=0.0, res_attention=False) :: Module

Scaled Dot-Product Attention module (Vaswani et al., 2017) with optional residual attention from previous layer (He et al, 2020)

{% endraw %} {% raw %}
{% endraw %} {% raw %}
B = 16
C = 10
M = 1500 # seq_len

n_heads = 1
D = 128 # model dimension
N = 512 # max_seq_len - latent's index dimension
d_k = D // n_heads

xb = torch.randn(B, C, M)
xb = (xb - xb.mean()) / xb.std()

# Attention
# input (Q)
lin = nn.Linear(M, N, bias=False)
Q = lin(xb).transpose(1,2)
test_eq(Q.shape, (B, N, C))

# q
to_q = nn.Linear(C, D, bias=False)
q = to_q(Q)
q = nn.LayerNorm(D)(q)

# k, v
context = xb.transpose(1,2)
to_kv = nn.Linear(C, D * 2, bias=False)
k, v = to_kv(context).chunk(2, dim = -1)
k = k.transpose(-1, -2)
k = nn.LayerNorm(M)(k)
v = nn.LayerNorm(D)(v)

test_eq(q.shape, (B, N, D))
test_eq(k.shape, (B, D, M))
test_eq(v.shape, (B, M, D))

output, attn, scores = ScaledDotProductAttention(res_attention=True)(q.unsqueeze(1), k.unsqueeze(1), v.unsqueeze(1))
test_eq(output.shape, (B, 1, N, D))
test_eq(attn.shape, (B, 1, N, M))
test_eq(scores.shape, (B, 1, N, M))
scores.mean(), scores.std()
{% endraw %} {% raw %}
# class MultiheadAttention(Module):
#     def __init__(self, d_model:int, n_heads:int, d_k:Optional[int]=None, d_v:Optional[int]=None, res_attention:bool=False, 
#                  dropout:float=0., qkv_bias:bool=True):
#         """Multi Head Attention Layer

#         Input shape:
#             Q:       [batch_size (bs) x max_q_len x d_model]
#             K, V:    [batch_size (bs) x q_len x d_model]
#             mask:    [q_len x q_len]
#         """

#         d_k = ifnone(d_k, d_model // n_heads)
#         d_v = ifnone(d_v, d_model // n_heads)

#         self.n_heads, self.d_k, self.d_v = n_heads, d_k, d_v

#         self.W_Q = nn.Linear(d_model, d_k * n_heads, bias=qkv_bias)
#         self.W_K = nn.Linear(d_model, d_k * n_heads, bias=qkv_bias)
#         self.W_V = nn.Linear(d_model, d_v * n_heads, bias=qkv_bias)

#         # Scaled Dot-Product Attention (multiple heads)
#         self.res_attention = res_attention
#         self.sdp_attn = ScaledDotProductAttention(res_attention=self.res_attention)

#         # Poject output
#         project_out = not (n_heads == 1 and d_model == d_k)
#         self.to_out = nn.Sequential(nn.Linear(n_heads * d_v, d_model), nn.Dropout(dropout)) if project_out else nn.Identity()


#     def forward(self, Q:Tensor, K:Optional[Tensor]=None, V:Optional[Tensor]=None, prev:Optional[Tensor]=None,
#                 key_padding_mask:Optional[Tensor]=None, attn_mask:Optional[Tensor]=None):

#         bs = Q.size(0)
#         if K is None: K = Q
#         if V is None: V = Q

#         # Linear (+ split in multiple heads)
#         q_s = self.W_Q(Q).view(bs, -1, self.n_heads, self.d_k).transpose(1,2)       # q_s    : [bs x n_heads x max_q_len x d_k]
#         k_s = self.W_K(K).view(bs, -1, self.n_heads, self.d_k).permute(0,2,3,1)     # k_s    : [bs x n_heads x d_k x q_len] - transpose(1,2) + transpose(2,3)
#         v_s = self.W_V(V).view(bs, -1, self.n_heads, self.d_v).transpose(1,2)       # v_s    : [bs x n_heads x q_len x d_v]

#         # Apply Scaled Dot-Product Attention (multiple heads)
#         if self.res_attention:
#             output, attn_weights, attn_scores = self.sdp_attn(q_s, k_s, v_s, prev=prev, key_padding_mask=key_padding_mask, attn_mask=attn_mask)
#         else:
#             output, attn_weights = self.sdp_attn(q_s, k_s, v_s, key_padding_mask=key_padding_mask, attn_mask=attn_mask)
#         # output: [bs x n_heads x q_len x d_v], attn: [bs x n_heads x q_len x q_len], scores: [bs x n_heads x max_q_len x q_len]

#         # back to the original inputs dimensions
#         output = output.transpose(1, 2).contiguous().view(bs, -1, self.n_heads * self.d_v) # output: [bs x q_len x n_heads * d_v]
#         output = self.to_out(output)

#         if self.res_attention: return output, attn_weights, attn_scores
#         else: return output, attn_weights 
{% endraw %} {% raw %}

class MultiheadAttention[source]

MultiheadAttention(d_model, n_heads, d_k=None, d_v=None, res_attention=False, attn_dropout=0.0, proj_dropout=0.0, qkv_bias=True) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
q = torch.rand([16, 3, 50, 8]) 
k = torch.rand([16, 3, 50, 8]).transpose(-1, -2)
v = torch.rand([16, 3, 50, 6])
attn_mask = torch.triu(torch.ones(50, 50)) # shape: q_len x q_len
key_padding_mask = torch.zeros(16, 50)
key_padding_mask[[1, 3, 6, 15], -10:] = 1
key_padding_mask = key_padding_mask.bool()
print('attn_mask', attn_mask.shape, 'key_padding_mask', key_padding_mask.shape)
output, attn = ScaledDotProductAttention(attn_dropout=.1)(q, k, v, attn_mask=attn_mask, key_padding_mask=key_padding_mask)
output.shape, attn.shape
{% endraw %} {% raw %}
t = torch.rand(16, 50, 128)
output, attn = MultiheadAttention(d_model=128, n_heads=3, d_k=8, d_v=6)(t, t, t, key_padding_mask=key_padding_mask, attn_mask=attn_mask)
output.shape, attn.shape
{% endraw %} {% raw %}
t = torch.rand(16, 50, 128)
att_mask = (torch.rand((50, 50)) > .85).float()
att_mask[att_mask == 1] = -np.inf

mha = MultiheadAttention(d_model=128, n_heads=3, d_k=8, d_v=6)
output, attn = mha(t, t, t, attn_mask=att_mask)
test_eq(torch.isnan(output).sum().item(), 0)
test_eq(torch.isnan(attn).sum().item(), 0)
loss = output[:2, :].sum()
test_eq(torch.isnan(loss).sum().item(), 0)
loss.backward()
for n, p in mha.named_parameters(): test_eq(torch.isnan(p.grad).sum().item(), 0)
{% endraw %} {% raw %}
t = torch.rand(16, 50, 128)
attn_mask = (torch.rand((50, 50)) > .85)

# True values will be masked
mha = MultiheadAttention(d_model=128, n_heads=3, d_k=8, d_v=6)
output, attn = mha(t, t, t, attn_mask=att_mask)
test_eq(torch.isnan(output).sum().item(), 0)
test_eq(torch.isnan(attn).sum().item(), 0)
loss = output[:2, :].sum()
test_eq(torch.isnan(loss).sum().item(), 0)
loss.backward()
for n, p in mha.named_parameters(): test_eq(torch.isnan(p.grad).sum().item(), 0)
{% endraw %} {% raw %}

class MultiConv1d[source]

MultiConv1d(ni, nf=None, kss=[1, 3, 5, 7], keep_original=False, separable=False, dim=1, **kwargs) :: Module

Module that applies multiple convolutions with different kernel sizes

{% endraw %} {% raw %}
{% endraw %} {% raw %}
t = torch.rand(16, 6, 37)
test_eq(MultiConv1d(6, None, kss=[1,3,5], keep_original=True)(t).shape, [16, 24, 37])
test_eq(MultiConv1d(6, 36, kss=[1,3,5], keep_original=False)(t).shape, [16, 36, 37])
test_eq(MultiConv1d(6, None, kss=[1,3,5], keep_original=True, dim=-1)(t).shape, [16, 6, 37*4])
test_eq(MultiConv1d(6, 60, kss=[1,3,5], keep_original=True)(t).shape, [16, 60, 37])
test_eq(MultiConv1d(6, 60, kss=[1,3,5], separable=True)(t).shape, [16, 60, 37])
{% endraw %} {% raw %}

class LSTMOutput[source]

LSTMOutput() :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
t = ([1], [2], [3])
test_eq(LSTMOutput()(t), [1])
{% endraw %} {% raw %}

class TSEmbedding[source]

TSEmbedding(ni, nf, std=0.01, padding_idx=None) :: Embedding

Embedding layer with truncated normal initialization adapted from fastai

{% endraw %} {% raw %}
{% endraw %} {% raw %}

class MultiEmbedding[source]

MultiEmbedding(c_in, n_embeds, embed_dims=None, cat_pos=None, std=0.01, padding_idx=0) :: Module

Same as nn.Module, but no need for subclasses to call super().__init__

{% endraw %} {% raw %}
{% endraw %} {% raw %}
a = alphabet[np.random.randint(0,3,40)]
b = ALPHABET[np.random.randint(6,10,40)]
c = np.random.rand(40).reshape(4,1,10)
map_a = {k:v for v,k in enumerate(np.unique(a))}
map_b = {k:v for v,k in enumerate(np.unique(b))}
n_embeds = [len(m.keys()) for m in [map_a, map_b]]
szs = [emb_sz_rule(n) for n in n_embeds]
a = np.asarray(a.map(map_a)).reshape(4,1,10)
b = np.asarray(b.map(map_b)).reshape(4,1,10)
inp = torch.from_numpy(np.concatenate((c,a,b), 1)).float()
memb = MultiEmbedding(3, n_embeds, cat_pos=[1,2])
# registered buffers are part of the state_dict() but not module.parameters()
assert all([(k in memb.state_dict().keys()) for k in ['cat_pos', 'cont_pos']])
embeddings = memb(inp)
print(n_embeds, szs, inp.shape, embeddings.shape)
test_eq(embeddings.shape, (inp.shape[0],sum(szs)+1,inp.shape[-1]))
{% endraw %}