class TSSequencerPlus(nn.Sequential):
r"""Time Series Sequencer model based on:
Tatsunami, Y., & Taki, M. (2022). Sequencer: Deep LSTM for Image Classification. arXiv preprint arXiv:2205.01972.
Official implementation: https://github.com/okojoalg/sequencer
Args:
c_in: the number of features (aka variables, dimensions, channels) in the time series dataset.
c_out: the number of target classes.
seq_len: number of time steps in the time series.
d_model: total dimension of the model (number of features created by the model).
depth: number of blocks in the encoder.
act: the activation function of positionwise feedforward layer.
lstm_dropout: dropout rate applied to the lstm sublayer.
dropout: dropout applied to to the embedded sequence steps after position embeddings have been added and
to the mlp sublayer in the encoder.
drop_path_rate: stochastic depth rate.
mlp_ratio: ratio of mlp hidden dim to embedding dim.
lstm_bias: determines whether bias is applied to the LSTM layer.
pre_norm: if True normalization will be applied as the first step in the sublayers. Defaults to False.
use_token: if True, the output will come from the transformed token. This is meant to be use in classification tasks.
use_pe: flag to indicate if positional embedding is used.
n_cat_embeds: list with the sizes of the dictionaries of embeddings (int).
cat_embed_dims: list with the sizes of each embedding vector (int).
cat_padding_idxs: If specified, the entries at cat_padding_idxs do not contribute to the gradient; therefore, the embedding vector at cat_padding_idxs
are not updated during training. Use 0 for those categorical embeddings that may have #na# values. Otherwise, leave them as None.
You can enter a combination for different embeddings (for example, [0, None, None]).
cat_pos: list with the position of the categorical variables in the input.
token_size: Size of the embedding function used to reduce the sequence length (similar to ViT's patch size)
tokenizer: nn.Module or callable that will be used to reduce the sequence length
feature_extractor: nn.Module or callable that will be used to preprocess the time series before
the embedding step. It is useful to extract features or resample the time series.
flatten: flag to indicate if the 3d logits will be flattened to 2d in the model's head if use_token is set to False.
If use_token is False and flatten is False, the model will apply a pooling layer.
concat_pool: if True the head begins with fastai's AdaptiveConcatPool2d if concat_pool=True; otherwise, it uses traditional average pooling.
fc_dropout: dropout applied to the final fully connected layer.
use_bn: flag that indicates if batchnorm will be applied to the head.
bias_init: values used to initialized the output layer.
y_range: range of possible y values (used in regression tasks).
custom_head: custom head that will be applied to the network. It must contain all kwargs (pass a partial function)
verbose: flag to control verbosity of the model.
Input:
x: bs (batch size) x nvars (aka features, variables, dimensions, channels) x seq_len (aka time steps)
"""
def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=6, act:str='gelu',
lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, lstm_bias:bool=True,
pre_norm:bool=False, use_token:bool=False, use_pe:bool=True,
cat_pos:Optional[list]=None, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None, cat_padding_idxs:Optional[list]=None,
token_size:int=None, tokenizer:Optional[Callable]=None, feature_extractor:Optional[Callable]=None,
flatten:bool=False, concat_pool:bool=True, fc_dropout:float=0., use_bn:bool=False,
bias_init:Optional[Union[float, list]]=None, y_range:Optional[tuple]=None, custom_head:Optional[Callable]=None, verbose:bool=True,
**kwargs):
if use_token and c_out == 1:
use_token = False
pv("use_token set to False as c_out == 1", verbose)
backbone = _TSSequencerBackbone(c_in, seq_len, depth=depth, d_model=d_model, act=act,
lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=drop_path_rate,
pre_norm=pre_norm, mlp_ratio=mlp_ratio, use_pe=use_pe, use_token=use_token,
n_cat_embeds=n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos,
feature_extractor=feature_extractor, token_size=token_size, tokenizer=tokenizer)
self.head_nf = d_model
self.c_out = c_out
self.seq_len = seq_len
# Head
if custom_head:
if isinstance(custom_head, nn.Module): head = custom_head
else: head = custom_head(self.head_nf, c_out, seq_len, **kwargs)
else:
nf = d_model
layers = []
if use_token:
layers += [TokenLayer()]
elif flatten:
layers += [Reshape(-1)]
nf = nf * seq_len
else:
if concat_pool: nf *= 2
layers = [GACP1d(1) if concat_pool else GAP1d(1)]
if use_bn: layers += [nn.BatchNorm1d(nf)]
if fc_dropout: layers += [nn.Dropout(fc_dropout)]
# Last layer
linear = nn.Linear(nf, c_out)
if bias_init is not None:
if isinstance(bias_init, float): nn.init.constant_(linear.bias, bias_init)
else: linear.bias = nn.Parameter(torch.as_tensor(bias_init, dtype=torch.float32))
layers += [linear]
if y_range: layers += [SigmoidRange(*y_range)]
head = nn.Sequential(*layers)
super().__init__(OrderedDict([('backbone', backbone), ('head', head)]))
TSSequencer = TSSequencerPlus
