<kaggle_start><code>import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os

for dirname, _, filenames in os.walk("/kaggle/input"):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# Any results you write to the current directory are saved as output.
import os


def log(*args):
    os.system(f'echo "{args}"')
    print(*args)


# In the [Data preparation notebook](https://www.kaggle.com/tchaye59/data-preparation), we've prepared the dataset in a suitable format ready to feed into a machine learning pipeline. In this one, we will use tf.data API to build our transformations pipeline and feed our network.
# ### We install the TFDS library and import required modules:
import tensorflow as tf
from tensorflow import keras
import pickle
import pandas
import numpy as np
from matplotlib import pyplot as plt
import gc
import threading
import time
import tensorflow_datasets.public_api as tfds
from tensorflow.keras import backend as K
from sklearn import metrics

tf.random.set_seed(1234)
np.random.seed(1234)
# The columns give us an idea of what our data looks like
columns = pickle.load(open("/kaggle/input/data-preparation/columns.pkl", "rb"))
max_day = 1941
print(f"Columns-{len(columns)} : {columns}")
print("Target: ", columns[5])
target_col = 5


# ## Let define some utils
# This function is just a wrapper to a function that calls add_time_steps
def preprocess(
    lookback,
    delay,
    target_col=target_col,
    lookback_step=1,
    test=False,
    val_days=0,
    return_key=False,
    val=False,
):
    # It takes as input a single time series data and applies some transformations to add time step
    def fn(inputs):
        # Remember _generate_examples of MyFullDataset we yielded a dictionary containing key and input
        values = inputs["input"]
        key = inputs["key"]
        return add_time_steps_ntarget(
            (key, values),
            lookback,
            delay,
            lookback_step=lookback_step,
            target_col=target_col,
            val_days=val_days,
            val=val,
            test=test,
            return_key=return_key,
        )

    return fn


# This function takes an item with all it 1941 days salles, prices, and calendar data adds lookback
# and generate the inputs and targets.
def add_time_steps(
    inputs,
    lookback,
    delay,
    target_col=target_col,
    test=False,
    lookback_step=1,
    val=False,
    val_days=0,
    return_key=False,
):
    key, values = inputs
    max_index = values.shape[0] - 1
    min_index = 0
    y = None
    idx = tf.signal.frame(tf.range(min_index, max_index), lookback, lookback_step)
    if not test:
        idx = idx[tf.reduce_all(idx + delay <= max_index, axis=-1)]
        if val:
            idx = idx[-val_days:]
        else:
            if val_days:
                idx = idx[:-val_days]
        y_idx = idx[..., -1] + delay
        y = tf.gather(values, y_idx)[..., target_col]
    else:
        idx = idx[-delay:]
    X = tf.gather(values, idx)
    if not test and return_key:
        return (key, X, y)
    return (X, y) if not test else (key, X)


# This function takes an item with all it 1941 days salles, prices, and calendar data adds lookback
# and generate the inputs and targets.
def add_time_steps_ntarget(
    inputs,
    lookback,
    delay,
    target_col=target_col,
    test=False,
    lookback_step=1,
    val=False,
    val_days=0,
    return_key=False,
):
    key, values = inputs
    max_index = values.shape[0] - 1
    min_index = 0
    val_steps = val_days // delay
    y = None
    idx = tf.signal.frame(tf.range(min_index, max_index), lookback, lookback_step)
    if not test:
        y_idx = idx[:, -1]
        y_idx = tf.map_fn(lambda x: tf.range(x, x + delay), y_idx)
        select = tf.reduce_all(y_idx <= max_index, axis=-1)
        idx = idx[select]
        y_idx = y_idx[select]
        if val:
            y_idx = y_idx[-val_steps:]
            idx = idx[-val_steps:]
        else:
            if val_days:
                idx = idx[:-val_steps]
                y_idx = y_idx[:-val_steps]
        y = tf.gather(values[..., target_col], y_idx)
    else:
        idx = idx[-1:]
    X = tf.gather(values, idx)
    if not test and return_key:
        return (key, X, y)
    return (X, y) if not test else (key, X)


# We need again this class
class MyFullDataset(tfds.core.GeneratorBasedBuilder):
    VERSION = tfds.core.Version("0.1.0")

    def _split_generators(self, dl_manager):
        return [
            tfds.core.SplitGenerator(
                name=f"train",
                gen_kwargs={},
            )
        ]

    def _info(self):
        shape = (max_day, len(columns))
        return tfds.core.DatasetInfo(
            builder=self,
            description=(""),
            features=tfds.features.FeaturesDict(
                {
                    "input": tfds.features.Tensor(shape=shape, dtype=tf.float32),
                    "key": tfds.features.Tensor(shape=(), dtype=tf.int32),
                }
            ),
        )

    def _generate_examples(self, **args):
        # We no longer need this function because we already build our dataset
        pass


# Now we are ready to build our pipelines 😋️
# ###  Let create our source
# The code is the same as what we did to prepare the dataset but since the dataset is already available TFDS will not try to create it
log("Download dataset")
gs_path = "gs://bucket59"
builder = MyFullDataset(data_dir=gs_path)
builder.download_and_prepare()
dataset_ = builder.as_dataset()["train"].repeat()
# Amazing right look at your dataset info : we only have one split named train.
# No need to worry we will use our preprocess and build differents pipelines to access the training, validation and test data from this single split
log(builder.info)

# This an example of how to access split's data by getting it as a tf.data.Dataset object
log("Try data access")
dataset = builder.as_dataset()["train"]
for item in dataset.take(1):
    # We access 2 time-series
    log("key: ", item["key"])
    log("input: ", item["input"].shape)
# We apply some transformations: add lookback and get the input and target
lookback = 5
delay = 28
preprocessor = preprocess(lookback, delay)
dataset = dataset.map(preprocessor)
for X, y in dataset.take(1):
    log(X.shape, y.shape)
# del dataset
gc.collect()

# ## Prepare for training
val_days = 28  # we use last 28 days of each time step for validation
lookback = 28  # time steps
lookback_step = 1
delay = 28  # We we are forecasting 28 days in the future
batch_size = 2**11
buffer_size = batch_size * 100
prefetch = 100  #
total_num_examples = 30490
ds_name = "train"  # The split name
load_weights = False
train = False
log(batch_size)


# Since our dataset is not normalized, we need  our first layer to be a normalization layer
def build_model():
    input_ = keras.layers.Input(
        shape=(
            lookback,
            len(columns),
        )
    )
    bn = keras.layers.BatchNormalization()(input_)
    lstm = keras.layers.Bidirectional(
        keras.layers.LSTM(256, return_sequences=True, recurrent_dropout=0.1)
    )(bn)
    lstm = keras.layers.Bidirectional(keras.layers.LSTM(256, recurrent_dropout=0.1))(
        lstm
    )
    dense = keras.layers.Dense(delay, activation=keras.activations.relu)(lstm)
    dense = keras.layers.Activation("relu")(dense)
    model = keras.models.Model(input_, dense)
    log(model.summary())
    model.compile(
        optimizer=keras.optimizers.Adam(0.01),
        loss=keras.losses.mean_squared_error,
        metrics=["mse", tf.keras.metrics.RootMeanSquaredError()],
    )
    return model


# detect and init the TPU
tpu_strategy = None
try:
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
    tf.config.experimental_connect_to_cluster(tpu)
    tf.tpu.experimental.initialize_tpu_system(tpu)
    log("Running on TPU ", tpu.cluster_spec().as_dict()["worker"])
    # instantiate a distribution strategy
    tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)
except:
    pass
if tpu_strategy:
    with tpu_strategy.scope():
        model = build_model()
else:
    model = build_model()
# Let Try to load weight from the previous commit
if load_weights and os.path.exists("/kaggle/input/modelh5/model.h5"):
    log("Loading weights")
    # model.load_weights('/kaggle/input/training/model.h5')
    model.load_weights("/kaggle/input/modelh5/model.h5")


# ### Training & Validation pipelines
#
# This pipeline provides training dataset
preprocessor = preprocess(
    lookback, delay, lookback_step=lookback_step, val_days=val_days
)
train_dataset = (
    dataset_.take(total_num_examples)
    .repeat()
    .shuffle(buffer_size=1000)
    .map(preprocessor)
    .unbatch()
)
train_dataset = (
    train_dataset.shuffle(buffer_size=buffer_size).batch(batch_size).prefetch(prefetch)
)
train_steps = (
    (total_num_examples * max_day - val_days - delay) // batch_size // lookback_step
)
log(train_steps)
# This one provides validation dataset
val_preprocessor = preprocess(
    lookback, delay, val_days=val_days, val=True, lookback_step=lookback_step
)
val_dataset = dataset_.take(total_num_examples).repeat().map(val_preprocessor).unbatch()
val_dataset = val_dataset.batch(total_num_examples).prefetch(prefetch)
val_steps = val_days // delay  # (total_num_examples*val_days)//batch_size
log(val_steps)

# # Training
if train:
    log("Start training")
    history = model.fit(
        train_dataset,
        steps_per_epoch=train_steps,
        epochs=30,
        validation_data=val_dataset,
        validation_steps=val_steps,
        callbacks=[
            keras.callbacks.EarlyStopping(
                monitor="loss",
                patience=50,
                restore_best_weights=True,
            ),
            keras.callbacks.ModelCheckpoint(
                filepath=f"model.h5",
                monitor="val_loss",
                save_best_only=True,
                save_weights_only=True,
            ),
            keras.callbacks.ReduceLROnPlateau(
                monitor="val_loss",
                factor=0.5,
                patience=10,
                min_lr=0.001,
            ),
        ],
    )
if train:
    pd.DataFrame(history.history).plot(figsize=(15, 8))
del train_dataset
del val_dataset
gc.collect()
model.load_weights(f"model.h5")
# Evaluate our validation data with the leaderboard
df_val = pd.read_csv(
    "/kaggle/input/m5-forecasting-accuracy/sample_submission.csv", index_col=0
)
df_val.iloc[total_num_examples:] = 0  # set evaluation to 0
val_preprocessor = preprocess(
    lookback,
    delay,
    val_days=val_days,
    val=True,
    lookback_step=lookback_step,
    return_key=True,
)
# fn_predict takes time-series data and generates predictions
fn_predict = lambda k, x, y: (k, model(x), y)
dataset = dataset_.take(total_num_examples)
dataset = dataset.map(
    val_preprocessor, num_parallel_calls=tf.data.experimental.AUTOTUNE
).prefetch(1000)
dataset = dataset.map(
    fn_predict, num_parallel_calls=tf.data.experimental.AUTOTUNE
).batch(total_num_examples)
for keys, y_pred, y in dataset:
    log(f"y_pred: {y_pred.shape}, y: {y.shape}, keys: {keys.shape}")
    y_pred = y_pred.numpy()
keys = np.argsort(keys.numpy())  # retrieve items ordering
df_val.iloc[:total_num_examples, :] = y_pred[keys]
df_val.to_csv("validation_submission.csv")


def generate_submission(
    model, lookback, lookback_step, delay, out_path="submission.csv"
):
    preprocessor = preprocess(lookback, delay, lookback_step=lookback_step, test=True)
    df_sub = pd.read_csv(
        "/kaggle/input/m5-forecasting-accuracy/sample_submission.csv", index_col=0
    )
    df_sub.iloc[:30490, :] = (
        pd.read_csv(
            "/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv",
            index_col=0,
        )
        .iloc[:, -28:]
        .values
    )
    fn_predict = lambda k, x: (k, model(x))
    dataset = dataset_.take(total_num_examples)
    dataset = dataset.map(
        preprocessor, num_parallel_calls=tf.data.experimental.AUTOTUNE
    ).prefetch(1000)
    dataset = dataset.map(
        fn_predict, num_parallel_calls=tf.data.experimental.AUTOTUNE
    ).batch(30490)
    for keys, y in dataset:
        y = y.numpy()
        log("y: ", y.shape)
    keys = np.argsort(keys.numpy())
    df_sub.iloc[total_num_examples:, :] = y[keys]
    df_sub.to_csv(out_path)
    del dataset


log("Generate Submission")
generate_submission(model, lookback, lookback_step, delay)
<|endoftext|>
[{"Id": 35900759, "ScriptId": 9817381, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 3699565, "CreationDate": "06/10/2020 17:34:57", "VersionNumber": 23.0, "Title": "M5-acc lstm model", "EvaluationDate": "06/10/2020", "IsChange": true, "TotalLines": 349.0, "LinesInsertedFromPrevious": 76.0, "LinesChangedFromPrevious": 0.0, "LinesUnchangedFromPrevious": 273.0, "LinesInsertedFromFork": NaN, "LinesDeletedFromFork": NaN, "LinesChangedFromFork": NaN, "LinesUnchangedFromFork": NaN, "TotalVotes": 0}]
import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os

for dirname, _, filenames in os.walk("/kaggle/input"):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# Any results you write to the current directory are saved as output.
import os


def log(*args):
    os.system(f'echo "{args}"')
    print(*args)


# In the [Data preparation notebook](https://www.kaggle.com/tchaye59/data-preparation), we've prepared the dataset in a suitable format ready to feed into a machine learning pipeline. In this one, we will use tf.data API to build our transformations pipeline and feed our network.
# ### We install the TFDS library and import required modules:
import tensorflow as tf
from tensorflow import keras
import pickle
import pandas
import numpy as np
from matplotlib import pyplot as plt
import gc
import threading
import time
import tensorflow_datasets.public_api as tfds
from tensorflow.keras import backend as K
from sklearn import metrics

tf.random.set_seed(1234)
np.random.seed(1234)
# The columns give us an idea of what our data looks like
columns = pickle.load(open("/kaggle/input/data-preparation/columns.pkl", "rb"))
max_day = 1941
print(f"Columns-{len(columns)} : {columns}")
print("Target: ", columns[5])
target_col = 5


# ## Let define some utils
# This function is just a wrapper to a function that calls add_time_steps
def preprocess(
    lookback,
    delay,
    target_col=target_col,
    lookback_step=1,
    test=False,
    val_days=0,
    return_key=False,
    val=False,
):
    # It takes as input a single time series data and applies some transformations to add time step
    def fn(inputs):
        # Remember _generate_examples of MyFullDataset we yielded a dictionary containing key and input
        values = inputs["input"]
        key = inputs["key"]
        return add_time_steps_ntarget(
            (key, values),
            lookback,
            delay,
            lookback_step=lookback_step,
            target_col=target_col,
            val_days=val_days,
            val=val,
            test=test,
            return_key=return_key,
        )

    return fn


# This function takes an item with all it 1941 days salles, prices, and calendar data adds lookback
# and generate the inputs and targets.
def add_time_steps(
    inputs,
    lookback,
    delay,
    target_col=target_col,
    test=False,
    lookback_step=1,
    val=False,
    val_days=0,
    return_key=False,
):
    key, values = inputs
    max_index = values.shape[0] - 1
    min_index = 0
    y = None
    idx = tf.signal.frame(tf.range(min_index, max_index), lookback, lookback_step)
    if not test:
        idx = idx[tf.reduce_all(idx + delay <= max_index, axis=-1)]
        if val:
            idx = idx[-val_days:]
        else:
            if val_days:
                idx = idx[:-val_days]
        y_idx = idx[..., -1] + delay
        y = tf.gather(values, y_idx)[..., target_col]
    else:
        idx = idx[-delay:]
    X = tf.gather(values, idx)
    if not test and return_key:
        return (key, X, y)
    return (X, y) if not test else (key, X)


# This function takes an item with all it 1941 days salles, prices, and calendar data adds lookback
# and generate the inputs and targets.
def add_time_steps_ntarget(
    inputs,
    lookback,
    delay,
    target_col=target_col,
    test=False,
    lookback_step=1,
    val=False,
    val_days=0,
    return_key=False,
):
    key, values = inputs
    max_index = values.shape[0] - 1
    min_index = 0
    val_steps = val_days // delay
    y = None
    idx = tf.signal.frame(tf.range(min_index, max_index), lookback, lookback_step)
    if not test:
        y_idx = idx[:, -1]
        y_idx = tf.map_fn(lambda x: tf.range(x, x + delay), y_idx)
        select = tf.reduce_all(y_idx <= max_index, axis=-1)
        idx = idx[select]
        y_idx = y_idx[select]
        if val:
            y_idx = y_idx[-val_steps:]
            idx = idx[-val_steps:]
        else:
            if val_days:
                idx = idx[:-val_steps]
                y_idx = y_idx[:-val_steps]
        y = tf.gather(values[..., target_col], y_idx)
    else:
        idx = idx[-1:]
    X = tf.gather(values, idx)
    if not test and return_key:
        return (key, X, y)
    return (X, y) if not test else (key, X)


# We need again this class
class MyFullDataset(tfds.core.GeneratorBasedBuilder):
    VERSION = tfds.core.Version("0.1.0")

    def _split_generators(self, dl_manager):
        return [
            tfds.core.SplitGenerator(
                name=f"train",
                gen_kwargs={},
            )
        ]

    def _info(self):
        shape = (max_day, len(columns))
        return tfds.core.DatasetInfo(
            builder=self,
            description=(""),
            features=tfds.features.FeaturesDict(
                {
                    "input": tfds.features.Tensor(shape=shape, dtype=tf.float32),
                    "key": tfds.features.Tensor(shape=(), dtype=tf.int32),
                }
            ),
        )

    def _generate_examples(self, **args):
        # We no longer need this function because we already build our dataset
        pass


# Now we are ready to build our pipelines 😋️
# ###  Let create our source
# The code is the same as what we did to prepare the dataset but since the dataset is already available TFDS will not try to create it
log("Download dataset")
gs_path = "gs://bucket59"
builder = MyFullDataset(data_dir=gs_path)
builder.download_and_prepare()
dataset_ = builder.as_dataset()["train"].repeat()
# Amazing right look at your dataset info : we only have one split named train.
# No need to worry we will use our preprocess and build differents pipelines to access the training, validation and test data from this single split
log(builder.info)

# This an example of how to access split's data by getting it as a tf.data.Dataset object
log("Try data access")
dataset = builder.as_dataset()["train"]
for item in dataset.take(1):
    # We access 2 time-series
    log("key: ", item["key"])
    log("input: ", item["input"].shape)
# We apply some transformations: add lookback and get the input and target
lookback = 5
delay = 28
preprocessor = preprocess(lookback, delay)
dataset = dataset.map(preprocessor)
for X, y in dataset.take(1):
    log(X.shape, y.shape)
# del dataset
gc.collect()

# ## Prepare for training
val_days = 28  # we use last 28 days of each time step for validation
lookback = 28  # time steps
lookback_step = 1
delay = 28  # We we are forecasting 28 days in the future
batch_size = 2**11
buffer_size = batch_size * 100
prefetch = 100  #
total_num_examples = 30490
ds_name = "train"  # The split name
load_weights = False
train = False
log(batch_size)


# Since our dataset is not normalized, we need  our first layer to be a normalization layer
def build_model():
    input_ = keras.layers.Input(
        shape=(
            lookback,
            len(columns),
        )
    )
    bn = keras.layers.BatchNormalization()(input_)
    lstm = keras.layers.Bidirectional(
        keras.layers.LSTM(256, return_sequences=True, recurrent_dropout=0.1)
    )(bn)
    lstm = keras.layers.Bidirectional(keras.layers.LSTM(256, recurrent_dropout=0.1))(
        lstm
    )
    dense = keras.layers.Dense(delay, activation=keras.activations.relu)(lstm)
    dense = keras.layers.Activation("relu")(dense)
    model = keras.models.Model(input_, dense)
    log(model.summary())
    model.compile(
        optimizer=keras.optimizers.Adam(0.01),
        loss=keras.losses.mean_squared_error,
        metrics=["mse", tf.keras.metrics.RootMeanSquaredError()],
    )
    return model


# detect and init the TPU
tpu_strategy = None
try:
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
    tf.config.experimental_connect_to_cluster(tpu)
    tf.tpu.experimental.initialize_tpu_system(tpu)
    log("Running on TPU ", tpu.cluster_spec().as_dict()["worker"])
    # instantiate a distribution strategy
    tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)
except:
    pass
if tpu_strategy:
    with tpu_strategy.scope():
        model = build_model()
else:
    model = build_model()
# Let Try to load weight from the previous commit
if load_weights and os.path.exists("/kaggle/input/modelh5/model.h5"):
    log("Loading weights")
    # model.load_weights('/kaggle/input/training/model.h5')
    model.load_weights("/kaggle/input/modelh5/model.h5")


# ### Training & Validation pipelines
#
# This pipeline provides training dataset
preprocessor = preprocess(
    lookback, delay, lookback_step=lookback_step, val_days=val_days
)
train_dataset = (
    dataset_.take(total_num_examples)
    .repeat()
    .shuffle(buffer_size=1000)
    .map(preprocessor)
    .unbatch()
)
train_dataset = (
    train_dataset.shuffle(buffer_size=buffer_size).batch(batch_size).prefetch(prefetch)
)
train_steps = (
    (total_num_examples * max_day - val_days - delay) // batch_size // lookback_step
)
log(train_steps)
# This one provides validation dataset
val_preprocessor = preprocess(
    lookback, delay, val_days=val_days, val=True, lookback_step=lookback_step
)
val_dataset = dataset_.take(total_num_examples).repeat().map(val_preprocessor).unbatch()
val_dataset = val_dataset.batch(total_num_examples).prefetch(prefetch)
val_steps = val_days // delay  # (total_num_examples*val_days)//batch_size
log(val_steps)

# # Training
if train:
    log("Start training")
    history = model.fit(
        train_dataset,
        steps_per_epoch=train_steps,
        epochs=30,
        validation_data=val_dataset,
        validation_steps=val_steps,
        callbacks=[
            keras.callbacks.EarlyStopping(
                monitor="loss",
                patience=50,
                restore_best_weights=True,
            ),
            keras.callbacks.ModelCheckpoint(
                filepath=f"model.h5",
                monitor="val_loss",
                save_best_only=True,
                save_weights_only=True,
            ),
            keras.callbacks.ReduceLROnPlateau(
                monitor="val_loss",
                factor=0.5,
                patience=10,
                min_lr=0.001,
            ),
        ],
    )
if train:
    pd.DataFrame(history.history).plot(figsize=(15, 8))
del train_dataset
del val_dataset
gc.collect()
model.load_weights(f"model.h5")
# Evaluate our validation data with the leaderboard
df_val = pd.read_csv(
    "/kaggle/input/m5-forecasting-accuracy/sample_submission.csv", index_col=0
)
df_val.iloc[total_num_examples:] = 0  # set evaluation to 0
val_preprocessor = preprocess(
    lookback,
    delay,
    val_days=val_days,
    val=True,
    lookback_step=lookback_step,
    return_key=True,
)
# fn_predict takes time-series data and generates predictions
fn_predict = lambda k, x, y: (k, model(x), y)
dataset = dataset_.take(total_num_examples)
dataset = dataset.map(
    val_preprocessor, num_parallel_calls=tf.data.experimental.AUTOTUNE
).prefetch(1000)
dataset = dataset.map(
    fn_predict, num_parallel_calls=tf.data.experimental.AUTOTUNE
).batch(total_num_examples)
for keys, y_pred, y in dataset:
    log(f"y_pred: {y_pred.shape}, y: {y.shape}, keys: {keys.shape}")
    y_pred = y_pred.numpy()
keys = np.argsort(keys.numpy())  # retrieve items ordering
df_val.iloc[:total_num_examples, :] = y_pred[keys]
df_val.to_csv("validation_submission.csv")


def generate_submission(
    model, lookback, lookback_step, delay, out_path="submission.csv"
):
    preprocessor = preprocess(lookback, delay, lookback_step=lookback_step, test=True)
    df_sub = pd.read_csv(
        "/kaggle/input/m5-forecasting-accuracy/sample_submission.csv", index_col=0
    )
    df_sub.iloc[:30490, :] = (
        pd.read_csv(
            "/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv",
            index_col=0,
        )
        .iloc[:, -28:]
        .values
    )
    fn_predict = lambda k, x: (k, model(x))
    dataset = dataset_.take(total_num_examples)
    dataset = dataset.map(
        preprocessor, num_parallel_calls=tf.data.experimental.AUTOTUNE
    ).prefetch(1000)
    dataset = dataset.map(
        fn_predict, num_parallel_calls=tf.data.experimental.AUTOTUNE
    ).batch(30490)
    for keys, y in dataset:
        y = y.numpy()
        log("y: ", y.shape)
    keys = np.argsort(keys.numpy())
    df_sub.iloc[total_num_examples:, :] = y[keys]
    df_sub.to_csv(out_path)
    del dataset


log("Generate Submission")
generate_submission(model, lookback, lookback_step, delay)
