<kaggle_start><code># # **IMPORTING LIBRARIES**
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tifffile as tiff
import cv2
import tensorflow as tf
from tensorflow.keras import layers
from skimage.transform import resize
from sklearn.model_selection import train_test_split


# # **DATA GENERATOR**
class imageDataGen(tf.keras.utils.Sequence):
    def __init__(
        self, df, shuffle=True, training=True, imageSize=(512, 512), batchSize=8
    ):
        self.df = df
        self.shuffle = shuffle
        self.training = training
        self.imageSize = imageSize
        self.batchSize = batchSize
        self.N = len(self.df)

    def on_epoch_end(self):
        """
        This functions runs at end of every epoch.
        """
        if self.shuffle:
            self.df = self.df.sample(frac=1).reset_index(drop=True)

    def __getitem__(self, index):
        """
        input  : index value of batch
        output : returns the batch of training images and target images
        """
        batchDF = self.df[index * self.batchSize : (index + 1) * self.batchSize]
        X = self.__getInput(batchDF.id)
        Y = self.__getOutput(batchDF[["rle", "img_width", "img_height"]])
        return (X, Y)

    def __len__(self):
        """
        Calculates the length of data with batch size.
        """
        return self.N // self.batchSize

    # Public helper functions.
    def mask2rle(self, img):
        """
        input  : image
        output : return the run length encoding of given image
        """
        pixels = img.T.flatten()
        pixels = np.concatenate([[0], pixels, [0]])
        runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
        runs[1::2] -= runs[::2]
        return " ".join(str(x) for x in runs)

    def rle2img(self, mask_rle, width, height):
        """
        input  : run length encoding
        output : return the image of given run length encoding.
        """
        shape = (width, height)
        s = mask_rle.split()
        starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]
        starts -= 1
        ends = starts + lengths
        img = np.zeros(shape[0] * shape[1], dtype=np.uint8)
        for lo, hi in zip(starts, ends):
            img[lo:hi] = 1
        img = img.reshape(shape).T.reshape(shape[0], shape[1], 1)
        return resize(img, self.imageSize)

    # Private helper functions
    def __loadImage(self, imgId):
        """
        input  : image id
        output : return the array of tiff image
        """
        img = tiff.imread(
            "../input/hubmap-organ-segmentation/train_images/" + str(imgId) + ".tiff"
        )
        img = resize(img, self.imageSize)
        img = tf.image.rgb_to_grayscale(img)
        return img

    def __augument(self, image):
        """
        input  : image
        output : augumented image
        """
        pass

    def __getInput(self, imageIds):
        """
        input  : image ids of batch data
        output : return the batch of training images
        """
        images = imageIds.map(self.__loadImage)
        if self.training:
            # images = images.map(self.__augument)
            pass
        return images

    def __getOutput(self, masksDf):
        """
        input  : RLE of batch data
        output : return the batch of training label images
        """
        maskImages = masksDf.apply(
            lambda x: self.rle2img(x["rle"], x["img_width"], x["img_height"]), axis=1
        )
        return maskImages


# # **U-Net MArchitecture**
# It is the U-Net architecture with using same padding on all convolutional layers.
# The input is in grayscale and output layer is with the sigmoid activation function.
# Input shape of model  : (512,512,1)
# Output shape of model : (512,512,1)
def buildModel():
    Input = layers.Input(shape=(512, 512, 1))
    # Feature extraction layers by conv layers
    conv1 = layers.Conv2D(64, (3, 3), (1, 1), padding="same", activation="relu")(Input)
    conv1 = layers.Conv2D(64, (3, 3), (1, 1), padding="same", activation="relu")(conv1)
    pool1 = layers.MaxPooling2D()(conv1)
    conv2 = layers.Conv2D(128, (3, 3), (1, 1), padding="same", activation="relu")(pool1)
    conv2 = layers.Conv2D(128, (3, 3), (1, 1), padding="same", activation="relu")(conv2)
    pool2 = layers.MaxPooling2D()(conv2)
    conv3 = layers.Conv2D(256, (3, 3), (1, 1), padding="same", activation="relu")(pool2)
    conv3 = layers.Conv2D(256, (3, 3), (1, 1), padding="same", activation="relu")(conv3)
    pool3 = layers.MaxPooling2D()(conv3)
    conv4 = layers.Conv2D(512, (3, 3), (1, 1), padding="same", activation="relu")(pool3)
    conv4 = layers.Conv2D(512, (3, 3), (1, 1), padding="same", activation="relu")(conv4)
    pool4 = layers.MaxPooling2D()(conv4)
    conv5 = layers.Conv2D(1024, (3, 3), (1, 1), padding="same", activation="relu")(
        pool4
    )
    conv5 = layers.Conv2D(1024, (3, 3), (1, 1), padding="same", activation="relu")(
        conv5
    )
    # Creating a represented mask by deconv layers
    dconv4 = layers.Conv2DTranspose(
        512, (3, 3), (2, 2), padding="same", activation="relu"
    )(conv5)
    uconv4 = layers.Concatenate()([dconv4, conv4])
    uconv4 = layers.Conv2D(512, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv4
    )
    uconv4 = layers.Conv2D(512, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv4
    )
    dconv3 = layers.Conv2DTranspose(
        256, (3, 3), (2, 2), padding="same", activation="relu"
    )(uconv4)
    uconv3 = layers.Concatenate()([dconv3, conv3])
    uconv3 = layers.Conv2D(256, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv3
    )
    uconv3 = layers.Conv2D(256, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv3
    )
    dconv2 = layers.Conv2DTranspose(
        128, (3, 3), (2, 2), padding="same", activation="relu"
    )(uconv3)
    uconv2 = layers.Concatenate()([dconv2, conv2])
    uconv2 = layers.Conv2D(128, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv2
    )
    uconv2 = layers.Conv2D(128, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv2
    )
    dconv1 = layers.Conv2DTranspose(
        64, (3, 3), (2, 2), padding="same", activation="relu"
    )(uconv2)
    uconv1 = layers.Concatenate()([dconv1, conv1])
    uconv1 = layers.Conv2D(64, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv1
    )
    uconv1 = layers.Conv2D(64, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv1
    )
    # Final output layer
    Output = layers.Conv2D(1, (3, 3), (1, 1), padding="same", activation="sigmoid")(
        uconv1
    )
    model = tf.keras.Model(inputs=[Input], outputs=[Output])
    return model


# # **DATA LOADING**
DF = pd.read_csv("../input/hubmap-organ-segmentation/train.csv")
# Splitting the data for training and validation
trainDF, validDF = train_test_split(DF, test_size=0.2)
# Creating a train and validation generator
trainGen = imageDataGen(trainDF)
validGen = imageDataGen(validDF)

# # **DATA VISUALIZATION**
# Generating data from the data generator
batch1 = trainGen.__getitem__(0)
x = batch1[0].values[5].numpy()
y = batch1[1].values[5]
print(f"Training images shape : {x.shape}")
print(f"Target images shape : {y.shape}")
plt.figure(figsize=(5, 5))
plt.imshow(x)
plt.imshow(y, cmap="coolwarm", alpha=0.5)
<|endoftext|>
[{"Id": 100993508, "ScriptId": 28234493, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 5144785, "CreationDate": "07/16/2022 15:35:53", "VersionNumber": 1.0, "Title": "Hacking the Human Body [ Data Gen & Model Archi]", "EvaluationDate": "07/16/2022", "IsChange": true, "TotalLines": 211.0, "LinesInsertedFromPrevious": 211.0, "LinesChangedFromPrevious": 0.0, "LinesUnchangedFromPrevious": 0.0, "LinesInsertedFromFork": NaN, "LinesDeletedFromFork": NaN, "LinesChangedFromFork": NaN, "LinesUnchangedFromFork": NaN, "TotalVotes": 1}]
# # **IMPORTING LIBRARIES**
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tifffile as tiff
import cv2
import tensorflow as tf
from tensorflow.keras import layers
from skimage.transform import resize
from sklearn.model_selection import train_test_split


# # **DATA GENERATOR**
class imageDataGen(tf.keras.utils.Sequence):
    def __init__(
        self, df, shuffle=True, training=True, imageSize=(512, 512), batchSize=8
    ):
        self.df = df
        self.shuffle = shuffle
        self.training = training
        self.imageSize = imageSize
        self.batchSize = batchSize
        self.N = len(self.df)

    def on_epoch_end(self):
        """
        This functions runs at end of every epoch.
        """
        if self.shuffle:
            self.df = self.df.sample(frac=1).reset_index(drop=True)

    def __getitem__(self, index):
        """
        input  : index value of batch
        output : returns the batch of training images and target images
        """
        batchDF = self.df[index * self.batchSize : (index + 1) * self.batchSize]
        X = self.__getInput(batchDF.id)
        Y = self.__getOutput(batchDF[["rle", "img_width", "img_height"]])
        return (X, Y)

    def __len__(self):
        """
        Calculates the length of data with batch size.
        """
        return self.N // self.batchSize

    # Public helper functions.
    def mask2rle(self, img):
        """
        input  : image
        output : return the run length encoding of given image
        """
        pixels = img.T.flatten()
        pixels = np.concatenate([[0], pixels, [0]])
        runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
        runs[1::2] -= runs[::2]
        return " ".join(str(x) for x in runs)

    def rle2img(self, mask_rle, width, height):
        """
        input  : run length encoding
        output : return the image of given run length encoding.
        """
        shape = (width, height)
        s = mask_rle.split()
        starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]
        starts -= 1
        ends = starts + lengths
        img = np.zeros(shape[0] * shape[1], dtype=np.uint8)
        for lo, hi in zip(starts, ends):
            img[lo:hi] = 1
        img = img.reshape(shape).T.reshape(shape[0], shape[1], 1)
        return resize(img, self.imageSize)

    # Private helper functions
    def __loadImage(self, imgId):
        """
        input  : image id
        output : return the array of tiff image
        """
        img = tiff.imread(
            "../input/hubmap-organ-segmentation/train_images/" + str(imgId) + ".tiff"
        )
        img = resize(img, self.imageSize)
        img = tf.image.rgb_to_grayscale(img)
        return img

    def __augument(self, image):
        """
        input  : image
        output : augumented image
        """
        pass

    def __getInput(self, imageIds):
        """
        input  : image ids of batch data
        output : return the batch of training images
        """
        images = imageIds.map(self.__loadImage)
        if self.training:
            # images = images.map(self.__augument)
            pass
        return images

    def __getOutput(self, masksDf):
        """
        input  : RLE of batch data
        output : return the batch of training label images
        """
        maskImages = masksDf.apply(
            lambda x: self.rle2img(x["rle"], x["img_width"], x["img_height"]), axis=1
        )
        return maskImages


# # **U-Net MArchitecture**
# It is the U-Net architecture with using same padding on all convolutional layers.
# The input is in grayscale and output layer is with the sigmoid activation function.
# Input shape of model  : (512,512,1)
# Output shape of model : (512,512,1)
def buildModel():
    Input = layers.Input(shape=(512, 512, 1))
    # Feature extraction layers by conv layers
    conv1 = layers.Conv2D(64, (3, 3), (1, 1), padding="same", activation="relu")(Input)
    conv1 = layers.Conv2D(64, (3, 3), (1, 1), padding="same", activation="relu")(conv1)
    pool1 = layers.MaxPooling2D()(conv1)
    conv2 = layers.Conv2D(128, (3, 3), (1, 1), padding="same", activation="relu")(pool1)
    conv2 = layers.Conv2D(128, (3, 3), (1, 1), padding="same", activation="relu")(conv2)
    pool2 = layers.MaxPooling2D()(conv2)
    conv3 = layers.Conv2D(256, (3, 3), (1, 1), padding="same", activation="relu")(pool2)
    conv3 = layers.Conv2D(256, (3, 3), (1, 1), padding="same", activation="relu")(conv3)
    pool3 = layers.MaxPooling2D()(conv3)
    conv4 = layers.Conv2D(512, (3, 3), (1, 1), padding="same", activation="relu")(pool3)
    conv4 = layers.Conv2D(512, (3, 3), (1, 1), padding="same", activation="relu")(conv4)
    pool4 = layers.MaxPooling2D()(conv4)
    conv5 = layers.Conv2D(1024, (3, 3), (1, 1), padding="same", activation="relu")(
        pool4
    )
    conv5 = layers.Conv2D(1024, (3, 3), (1, 1), padding="same", activation="relu")(
        conv5
    )
    # Creating a represented mask by deconv layers
    dconv4 = layers.Conv2DTranspose(
        512, (3, 3), (2, 2), padding="same", activation="relu"
    )(conv5)
    uconv4 = layers.Concatenate()([dconv4, conv4])
    uconv4 = layers.Conv2D(512, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv4
    )
    uconv4 = layers.Conv2D(512, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv4
    )
    dconv3 = layers.Conv2DTranspose(
        256, (3, 3), (2, 2), padding="same", activation="relu"
    )(uconv4)
    uconv3 = layers.Concatenate()([dconv3, conv3])
    uconv3 = layers.Conv2D(256, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv3
    )
    uconv3 = layers.Conv2D(256, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv3
    )
    dconv2 = layers.Conv2DTranspose(
        128, (3, 3), (2, 2), padding="same", activation="relu"
    )(uconv3)
    uconv2 = layers.Concatenate()([dconv2, conv2])
    uconv2 = layers.Conv2D(128, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv2
    )
    uconv2 = layers.Conv2D(128, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv2
    )
    dconv1 = layers.Conv2DTranspose(
        64, (3, 3), (2, 2), padding="same", activation="relu"
    )(uconv2)
    uconv1 = layers.Concatenate()([dconv1, conv1])
    uconv1 = layers.Conv2D(64, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv1
    )
    uconv1 = layers.Conv2D(64, (3, 3), (1, 1), padding="same", activation="relu")(
        uconv1
    )
    # Final output layer
    Output = layers.Conv2D(1, (3, 3), (1, 1), padding="same", activation="sigmoid")(
        uconv1
    )
    model = tf.keras.Model(inputs=[Input], outputs=[Output])
    return model


# # **DATA LOADING**
DF = pd.read_csv("../input/hubmap-organ-segmentation/train.csv")
# Splitting the data for training and validation
trainDF, validDF = train_test_split(DF, test_size=0.2)
# Creating a train and validation generator
trainGen = imageDataGen(trainDF)
validGen = imageDataGen(validDF)

# # **DATA VISUALIZATION**
# Generating data from the data generator
batch1 = trainGen.__getitem__(0)
x = batch1[0].values[5].numpy()
y = batch1[1].values[5]
print(f"Training images shape : {x.shape}")
print(f"Target images shape : {y.shape}")
plt.figure(figsize=(5, 5))
plt.imshow(x)
plt.imshow(y, cmap="coolwarm", alpha=0.5)
