Informer For Prediction, Need help!

[furkanbicer]

Hello everyone,

I am working on Informer for Prediction model from this link; Informer

I also attached small part of my training data. In the csv file, first column is timestamp and second column is my target. Other 82 column is my input data for every timestamp, my goal is using 10 timestamp for every prediction. My code to create and train informer model is here;

from transformers import InformerConfig

config = InformerConfig(
    d_model=128,  # dimensionality of the model
    n_heads=8,  # number of attention heads
    encoder_layers=3,  # number of encoder layers
    decoder_layers=3,  # number of decoder layers
    input_size=81,  # number of input features
    context_length=100,  # number of past timesteps
    prediction_length=1  # number of future timesteps to predict
)

import torch
from torch.optim import Adam
from transformers import InformerForPrediction

# Setup device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Creating tensor from data
#data = torch.tensor(data)

# Create a tensor with all ones
past_observed_mask = torch.ones(10)

# Initialize the model
model = InformerForPrediction(config).to(device)

# Define loss and optimizer
criterion = torch.nn.MSELoss()  # Mean Squared Error Loss for numerical prediction
optimizer = Adam(model.parameters(), lr=0.001)

# Training loop
for epoch in range(2):  # num_epochs is the number of epochs you want to train
    model.train()
    total_loss = 0
    for batch in dataloader:  # assuming dataloader is defined as before
        optimizer.zero_grad()
        # Get input and targets and move to GPU if available
        sequences, labels = batch
        sequences, labels = sequences.to(device), labels.to(device)

        past_values = sequences[:, :, 2:]  # Assuming the last column is the future value
        past_time_features = sequences[:, :, :2]  # Replace with the correct slice for time features

        past_observed_mask = torch.ones_like(past_values, dtype=torch.float)

        print(past_values, 'target', labels.unsqueeze(-1), 'past_time', past_time_features, 'mask', past_observed_mask)

        if torch.isnan(past_values).any() or torch.isnan(labels).any() or torch.isnan(past_time_features).any() or torch.isnan(past_observed_mask).any():
            print("past_time_features has empty or missing values.")
        else:
            print("past_time_features does not have empty or missing values.")

        # Forward pass
        outputs = model(
            past_values=past_values,
            future_values=labels,  # may need to adjust shape
            past_time_features=past_time_features,
            past_observed_mask=past_observed_mask
        )

        loss = criterion(outputs.last_hidden_state, labels)

        # Backward pass and optimize
        loss.backward()
        optimizer.step()

        total_loss += loss.item()
    print(f"Epoch {epoch+1}/{2}, Loss: {total_loss/len(dataloader)}")

But even though there is no None value in any tensor, I am getting this error;

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-25-0943f1126085> in <cell line: 22>()
     42 
     43         # Forward pass
---> 44         outputs = model(
     45             past_values=past_values,
     46             future_values=labels,  # may need to adjust shape

6 frames
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
   1516             return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517         else:
-> 1518             return self._call_impl(*args, **kwargs)
   1519 
   1520     def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
   1525                 or _global_backward_pre_hooks or _global_backward_hooks
   1526                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527             return forward_call(*args, **kwargs)
   1528 
   1529         try:

/usr/local/lib/python3.10/dist-packages/transformers/models/informer/modeling_informer.py in forward(self, past_values, past_time_features, past_observed_mask, static_categorical_features, static_real_features, future_values, future_time_features, future_observed_mask, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, output_hidden_states, output_attentions, use_cache, return_dict)
   1818             use_cache = False
   1819 
-> 1820         outputs = self.model(
   1821             past_values=past_values,
   1822             past_time_features=past_time_features,

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
   1516             return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517         else:
-> 1518             return self._call_impl(*args, **kwargs)
   1519 
   1520     def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
   1525                 or _global_backward_pre_hooks or _global_backward_hooks
   1526                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527             return forward_call(*args, **kwargs)
   1528 
   1529         try:

/usr/local/lib/python3.10/dist-packages/transformers/models/informer/modeling_informer.py in forward(self, past_values, past_time_features, past_observed_mask, static_categorical_features, static_real_features, future_values, future_time_features, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, output_hidden_states, output_attentions, use_cache, return_dict)
   1640         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
   1641 
-> 1642         transformer_inputs, loc, scale, static_feat = self.create_network_inputs(
   1643             past_values=past_values,
   1644             past_time_features=past_time_features,

/usr/local/lib/python3.10/dist-packages/transformers/models/informer/modeling_informer.py in create_network_inputs(self, past_values, past_time_features, static_categorical_features, static_real_features, past_observed_mask, future_values, future_time_features)
   1514         # time feature
   1515         time_feat = (
-> 1516             torch.cat(
   1517                 (
   1518                     past_time_features[:, self._past_length - self.config.context_length :, ...],

TypeError: expected Tensor as element 1 in argument 0, but got NoneType

Can someone help me about this please? What is my mistake and how can I fix this?

[nielsr]

cc @kashif

[kashif]

so I believe the issue is that the temporal inputs to the model have to have a temporal size that allows one to create lag features… since you didn’t specify the lag indices the default one is set: [1, 2, 3, 4, 5, 6, 7] which means that the input have to have 7 more time points (in the past)… ideally you want to set the lag indices based on the frequency of your time series

[furkanbicer]

@kashif I tried to pass more time steps to model (from 10 to 20 to be exact) and this is current situation of the code;

from transformers import InformerConfig

config = InformerConfig(
    d_model=128,  # dimensionality of the model
    n_heads=8,  # number of attention heads
    encoder_layers=3,  # number of encoder layers
    decoder_layers=3,  # number of decoder layers
    input_size=82,  # number of input features
    context_length=20,  # number of past timesteps
    prediction_length=1  # number of future timesteps to predict
)

features = data.iloc[:, 3:].values  # select input columns
targets = data.iloc[:, 2].values  # select target column

# Create sequences
def create_sequences(data, sequence_length):
    sequences = []
    for i in range(len(data) - sequence_length):
        sequences.append(data[i:i + sequence_length])
    return np.array(sequences)

df = pd.DataFrame()

# Convert Unix timestamps to datetime and extract time features
df['datetime'] = pd.to_datetime(data['timestamp'], unit='ms')  # replace with your timestamp column
df['month'] = df['datetime'].dt.month
df['day'] = df['datetime'].dt.day

# Append time features to your inputs
features_with_time = np.hstack((df[['month', 'day']].values, features))  # include all extracted time features

# Create sequences with the updated features
input_sequences = create_sequences(features_with_time, 20)
target_sequences = create_sequences(targets, 20)

# Convert to PyTorch tensors
input_sequences = torch.tensor(input_sequences, dtype=torch.float32)
target_sequences = torch.tensor(target_sequences[:, -1], dtype=torch.float32)  # predicting next step

# Create TensorDataset and DataLoader
dataset = TensorDataset(input_sequences, target_sequences)
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

import torch
from torch.optim import Adam
from transformers import InformerForPrediction

# Setup device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Creating tensor from data
#data = torch.tensor(data)

# Create a tensor with all ones
past_observed_mask = torch.ones(10)

# Initialize the model
model = InformerForPrediction(config).to(device)

# Define loss and optimizer
criterion = torch.nn.MSELoss()  # Mean Squared Error Loss for numerical prediction
optimizer = Adam(model.parameters(), lr=0.001)

# Training loop
for epoch in range(2):  # num_epochs is the number of epochs you want to train
    model.train()
    total_loss = 0
    for batch in dataloader:  # assuming dataloader is defined as before
        optimizer.zero_grad()
        # Get input and targets and move to GPU if available
        sequences, labels = batch
        sequences, labels = sequences.to(device), labels.to(device)

        past_values = sequences[:, :, 2:]  # Assuming the last column is the future value
        past_time_features = sequences[:, :, :2]  # Replace with the correct slice for time features

        past_observed_mask = torch.ones_like(past_values, dtype=torch.float)

        print(past_values, 'target', labels.unsqueeze(-1), 'past_time', past_time_features, 'mask', past_observed_mask)

        if torch.isnan(past_values).any() or torch.isnan(labels).any() or torch.isnan(past_time_features).any() or torch.isnan(past_observed_mask).any():
            print("past_time_features has empty or missing values.")
        else:
            print("past_time_features does not have empty or missing values.")

        # Forward pass
        outputs = model(
            past_values=past_values,
            future_values=labels,  # may need to adjust shape
            past_time_features=past_time_features,
            past_observed_mask=past_observed_mask
        )

        loss = criterion(outputs.last_hidden_state, labels)

        # Backward pass and optimize
        loss.backward()
        optimizer.step()

        total_loss += loss.item()
    print(f"Epoch {epoch+1}/{2}, Loss: {total_loss/len(dataloader)}")

Looks like nothing changed in the error statement;

TypeError                                 Traceback (most recent call last)
<ipython-input-23-0943f1126085> in <cell line: 22>()
     42 
     43         # Forward pass
---> 44         outputs = model(
     45             past_values=past_values,
     46             future_values=labels,  # may need to adjust shape

6 frames
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
   1516             return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517         else:
-> 1518             return self._call_impl(*args, **kwargs)
   1519 
   1520     def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
   1525                 or _global_backward_pre_hooks or _global_backward_hooks
   1526                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527             return forward_call(*args, **kwargs)
   1528 
   1529         try:

/usr/local/lib/python3.10/dist-packages/transformers/models/informer/modeling_informer.py in forward(self, past_values, past_time_features, past_observed_mask, static_categorical_features, static_real_features, future_values, future_time_features, future_observed_mask, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, output_hidden_states, output_attentions, use_cache, return_dict)
   1818             use_cache = False
   1819 
-> 1820         outputs = self.model(
   1821             past_values=past_values,
   1822             past_time_features=past_time_features,

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
   1516             return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517         else:
-> 1518             return self._call_impl(*args, **kwargs)
   1519 
   1520     def _call_impl(self, *args, **kwargs):

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
   1525                 or _global_backward_pre_hooks or _global_backward_hooks
   1526                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527             return forward_call(*args, **kwargs)
   1528 
   1529         try:

/usr/local/lib/python3.10/dist-packages/transformers/models/informer/modeling_informer.py in forward(self, past_values, past_time_features, past_observed_mask, static_categorical_features, static_real_features, future_values, future_time_features, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, output_hidden_states, output_attentions, use_cache, return_dict)
   1640         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
   1641 
-> 1642         transformer_inputs, loc, scale, static_feat = self.create_network_inputs(
   1643             past_values=past_values,
   1644             past_time_features=past_time_features,

/usr/local/lib/python3.10/dist-packages/transformers/models/informer/modeling_informer.py in create_network_inputs(self, past_values, past_time_features, static_categorical_features, static_real_features, past_observed_mask, future_values, future_time_features)
   1514         # time feature
   1515         time_feat = (
-> 1516             torch.cat(
   1517                 (
   1518                     past_time_features[:, self._past_length - self.config.context_length :, ...],

TypeError: expected Tensor as element 1 in argument 0, but got NoneType

Can you please point what is my mistake? I also attached small part of my data gere, if it makes any difference. Or is there any clear example that i can use to learn from it? I could not find enough source about Informer model.

chunk (5).csv

[kashif]

@furkanbicer

_past_length

yeah so i believe that the size of the past_time_features is not correct. Have a look at the blog post: Probabilistic Time Series Forecasting with 🤗 Transformers that shows the appropriate sizes of the inputs based on the lag indices