KFC: Kinship Verification with Fair Contrastive loss and Multi-Task Learning
from torch.utils.data import Dataset
from keras.preprocessing import image
import numpy as np
from utils import np2tensor
from utils import np2tensor_int
import random
from torchvision import transforms
from torch.utils.data import DataLoader
import torch


class FIW(Dataset):
    def __init__(self,
                 sample_path,
                 transform=None):

        self.sample_path=sample_path
        self.transform=transform
        self.sample_list=self.load_sample()
        self.bias=0
        self.race_dict={'AA':np.array([0], dtype=np.long),'A':np.array([1], dtype=np.long),'C':np.array([2], dtype=np.long),'I':np.array([3], dtype=np.long),\
            'AA&AA':np.array([4], dtype=np.long),'AA&A':np.array([5], dtype=np.long),'AA&C':np.array([6], dtype=np.long),'AA&I':np.array([7], dtype=np.long),\
            'A&AA':np.array([8], dtype=np.long),'A&A':np.array([9], dtype=np.long),'A&C':np.array([10], dtype=np.long),'A&I':np.array([11], dtype=np.long),\
            'C&AA':np.array([12], dtype=np.long),'C&A':np.array([13], dtype=np.long),'C&C':np.array([14], dtype=np.long),'C&I':np.array([15], dtype=np.long),\
            'I&AA':np.array([16], dtype=np.long),'I&A':np.array([17], dtype=np.long),'I&C':np.array([18], dtype=np.long),'I&I':np.array([19], dtype=np.long)}


    def load_sample(self):
        sample_list= []
        f = open(self.sample_path, "r+", encoding='utf-8')
        while True:
            line = f.readline().replace('\n', '')
            if not line:
                break
            else:
                tmp = line.split(' ')
                if tmp[4] == "AA":
                    race = 0
                elif tmp[4] == "A":
                    race = 1
                elif tmp[4] == "C":
                    race = 2
                elif tmp[4] == "I":
                    race = 3

                if tmp[2] == "fs":
                    kinship = 0
                elif tmp[2] == "fd":
                    kinship = 1
                elif tmp[2] == "ms":
                    kinship = 2
                elif tmp[2] == "md":
                    kinship = 3
                elif tmp[2] == "fms":
                    kinship = 4
                elif tmp[2] == "fmd":
                    kinship = 5
                elif tmp[2] == "fsd":
                    kinship = 6
                elif tmp[2] == "msd":
                    kinship = 7
                sample_list.append([tmp[0], tmp[1], kinship, tmp[3], tmp[4]])
        f.close()
        return sample_list

    def __len__(self):
        return len(self.sample_list)

    def read_image(self, path):
        img = image.load_img(path, target_size=(112, 112))
        return img

    def set_bias(self,bias):
        self.bias=bias%self.__len__()

    def preprocess(self, img):
        return np.transpose(img, (2, 0, 1))

    def __getitem__(self, item):
        sample = self.sample_list[(item+self.bias)%self.__len__()]
        img1,img2=self.read_image(sample[0]),self.read_image(sample[1])
        if self.transform is not None:
            img1,img2 = self.transform(img1),self.transform(img2)
        img1, img2 = np2tensor(self.preprocess(np.array(img1, dtype=float))), \
                     np2tensor(self.preprocess(np.array(img2, dtype=float)))
        
        label = np2tensor(np.array(sample[3], dtype=float))
        race = torch.squeeze(np2tensor(self.race_dict[sample[4]])).to(torch.long)
        kinship = np2tensor_int(np.array(sample[2], dtype=int))

        return img1, img2, kinship, label, race

dataset.py
garynlfd/KFC
import gc
from sklearn.metrics import roc_curve,auc
from dataset import *
from models_multi_task import *
from utils import *
import torch_resnet101
import torch
from torch.optim import SGD
from losses import *
import sys
import argparse

def find(args):
    path=os.path.join(args.save_path,"weights_Cornell_multi")
    batch_size=args.batch_size
    log_path=args.log_path
    val_dataset = FIW(os.path.join(args.sample,"./aligned_dataset/mixed_dataset/mixed_dataset_val_cross.txt"))
    val_loader = DataLoader(val_dataset, batch_size=batch_size, num_workers=1, pin_memory=False)
    model = MultiTask().cuda()
    model.load(path, "_std_best")
    model.eval()
    with torch.no_grad():
        auc ,threshold = val_model(model, val_loader)
    mylog("auc : ",auc,path=log_path)
    mylog("threshold :" ,threshold,path=log_path)


def val_model(model, val_loader):
    y_true = []
    y_pred = []
    heartbeat = 0
    for img1, img2, kinship, labels, races in val_loader:
        r1,r2,e1,e2,x1,x2,bias_map,bias_pair=model([img1.cuda(),img2.cuda()])
        y_pred.extend(torch.cosine_similarity(e1,e2,dim=1).cpu().detach().numpy().tolist())
        y_true.extend(labels.cpu().detach().numpy().tolist())
        heartbeat += 1
        if(heartbeat % 100 == 0): print(heartbeat)
    fpr, tpr, thresholds_keras = roc_curve(y_true, y_pred)
    maxindex = (tpr - fpr).tolist().index(max(tpr - fpr))
    threshold = thresholds_keras[maxindex]
    return auc(fpr,tpr),threshold


if __name__=="__main__":
    parser = argparse.ArgumentParser(description="find threshold")
    parser.add_argument("--sample", type=str, help="sample root")
    parser.add_argument("--save_path", type=str, help="model save path")
    parser.add_argument("--batch_size", type=int, default=40, help="batch size default 40")
    parser.add_argument("--log_path", type=str, default="./log.txt",help="log path default log.txt ")
    parser.add_argument("--gpu", default="1", type=str, help="gpu id you use")
    args = parser.parse_args()
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
    torch.multiprocessing.set_start_method("spawn")
    set_seed(100)
    find(args)

find.py
garynlfd/KFC
from cmath import nan
import torch
import numpy as np
# from Track1.utils import *
from utils import *
import math

# only loss function
# def contrastive_loss(x1,x2,kinship,race,bias_map,bias_pair,beta=0.08):
#     x1x2=torch.cat([x1,x2],dim=0)
#     x2x1=torch.cat([x2,x1],dim=0)
    
#     cosine_mat=torch.cosine_similarity(torch.unsqueeze(x1x2,dim=1),
#                                        torch.unsqueeze(x1x2,dim=0),dim=2)/(beta)
#     mask=1.0-torch.eye(2*x1.size(0))
#     diagonal_cosine=torch.cosine_similarity(x1x2,x2x1,dim=1)
    
#     debais_margin=torch.sum(bias_map,axis=1)/len(bias_map)
#     numerators = torch.exp((diagonal_cosine-debais_margin)/(beta))
#     denominators=torch.sum(torch.exp(cosine_mat)*mask,dim=1)-torch.exp(diagonal_cosine/beta)+numerators # - x1x2/beta+ x1x2/beta+epsilon

#     return -torch.mean(torch.log(numerators)-torch.log(denominators),dim=0), [AA_margin, A_margin, C_margin, I_margin]

# loss function with bias of every race
def contrastive_loss(x1,x2,kinship,race,bias_map,bias_pair,beta=0.08):

    batch_size = x1.size(0)
    AA_num = 0
    A_num = 0
    C_num = 0
    I_num = 0
    AA_idx = 0
    A_idx = 0
    C_idx = 0
    I_idx = 0
    x1x2=torch.cat([x1,x2],dim=0)
    x2x1=torch.cat([x2,x1],dim=0)
    
    cosine_mat=torch.cosine_similarity(torch.unsqueeze(x1x2,dim=1),
                                       torch.unsqueeze(x1x2,dim=0),dim=2)/(beta)
    mask=1.0-torch.eye(2*x1.size(0))
    diagonal_cosine=torch.cosine_similarity(x1x2,x2x1,dim=1)
    
    debais_margin=torch.sum(bias_map,axis=1)/len(bias_map)
    for i in range(batch_size):
        if race[i] == 0:
            AA_num += debais_margin[i] + debais_margin[i+batch_size]
            AA_idx += 2
        elif race[i] == 1:
            A_num += debais_margin[i] + debais_margin[i+batch_size]
            A_idx += 2
        elif race[i] == 2:
            C_num += debais_margin[i] + debais_margin[i+batch_size]
            C_idx += 2
        elif race[i] == 3:
            I_num += debais_margin[i] + debais_margin[i+batch_size]
            I_idx += 2
    if AA_idx == 0:
        AA_margin = 0
    else:
        AA_margin = AA_num / AA_idx
    if A_idx == 0:
        A_margin = 0
    else:
        A_margin = A_num / A_idx
    if C_idx == 0:
        C_margin = 0
    else:
        C_margin = C_num / C_idx
    if I_idx == 0:
        I_margin = 0
    else:
        I_margin = I_num / I_idx
    numerators = torch.exp((diagonal_cosine-debais_margin)/(beta))
    denominators=torch.sum(torch.exp(cosine_mat)*mask,dim=1)-torch.exp(diagonal_cosine/beta)+numerators # - x1x2/beta+ x1x2/beta+epsilon

    return -torch.mean(torch.log(numerators)-torch.log(denominators),dim=0), [AA_margin, A_margin, C_margin, I_margin]
losses.py
garynlfd/KFC
import torch
from torch_resnet101 import *
import torch.nn as nn
import torch.nn.functional as F
import math
import os
from torch.autograd import Function

# vanilla model structure

# class Net(torch.nn.Module):
#     def __init__(self):
#         super(Net, self).__init__()
#         self.encoder=KitModel("./kit_resnet101.pkl")
#         self.Atten_module=Atten()


#     def forward(self, imgs):
#         img1,img2=imgs
#         embeding1,compact_feature1= self.encoder(img1) # each featur 16, 512x7x7
#         embeding2,compact_feature2=self.encoder(img2)
        
#         atten_em1,atten_em2,x1,x2=self.Atten_module(embeding1,compact_feature1,embeding2,compact_feature2)

#         return embeding1,embeding2,x1,x2


class ReverseLayer(Function):
    @staticmethod
    def forward(ctx,x,alpha):
        ctx.alpha=alpha
        return x.view_as(x) # make x be same shape as x
    
    @staticmethod
    def backward(ctx,*grad_output): #grad_output backward gradient 
        #result , = ctx.saved_tensors # the saving forward gradient
        #print(grad_output)
        return -(grad_output[0]) * ctx.alpha, None   # sine input is tuple
def grad_reverse(x,alpha=1.0):
    return ReverseLayer.apply(x,alpha)
# ctx: is a pytorch syntex that can save the variable in function

class BasicConv(nn.Module):
    def __init__(self, in_planes, out_planes, kernel_size, stride=1, padding=0, dilation=1, groups=1, relu=True, bn=True, bias=False):
        super(BasicConv, self).__init__()
        self.out_channels = out_planes
        self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias)
        self.bn = nn.BatchNorm2d(out_planes,eps=1e-5, momentum=0.01, affine=True) if bn else None
        self.relu = nn.ReLU() if relu else None

    def forward(self, x):
        x = self.conv(x)
        if self.bn is not None:
            x = self.bn(x)
        if self.relu is not None:
            x = self.relu(x)
        return x

class Flatten(torch.nn.Module):
    def forward(self, x):
        return x.view(x.size(0), -1)

class CBAM(torch.nn.Module):
    def __init__(self, gate_channels, reduction_ratio=16, pool_types=['avg', 'max']):
        super(CBAM, self).__init__()
        self.ChannelGate = ChannelGate(gate_channels)
        self.SpatialGate = SpatialGate()
    def forward(self, x):
        x_out = self.ChannelGate(x)
        x_out = self.SpatialGate(x_out)
        return x_out

class ChannelGate(torch.nn.Module):
    def __init__(self,in_channel,ratio=16):
        super().__init__()
        self.atten_module=nn.Sequential(
            Flatten(),
            nn.Linear(in_channel, in_channel // 16),
            nn.ReLU(),
            nn.Linear(in_channel // 16, in_channel)
        )
    def forward(self,x):
        channel_att_sum = None
        avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))
        channel_att_avg = self.atten_module(avg_pool)
        max_pool = F.max_pool2d(x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))
        channel_att_max = self.atten_module(max_pool)
        channel_att_sum = channel_att_avg + channel_att_max
        scale = torch.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(x)
        x = x * scale
        return x

class ChannelPool(torch.nn.Module):
    def forward(self, x):
        return torch.cat((torch.max(x, 1)[0].unsqueeze(1), torch.mean(x, 1).unsqueeze(1)), dim=1)

class SpatialGate(nn.Module):
    def __init__(self):
        super(SpatialGate, self).__init__()
        kernel_size = 7
        self.compress = ChannelPool()
        self.spatial = BasicConv(2, 1, kernel_size, stride=1, padding=(kernel_size-1) // 2, relu=False)
        self.flatten=nn.Sequential(
            nn.Flatten()
        )
    def forward(self, x):
        x_compress = self.compress(x)
        x_out = self.spatial(x_compress)
        scale = torch.sigmoid(x_out) # broadcasting
        x = x * scale
        return self.flatten(x)
    
class Atten(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.extract_flatten_feature=nn.Sequential(
            nn.AdaptiveAvgPool2d(1), # 7x7x512 -> 1x1x512
            nn.Conv2d(in_channels=512,kernel_size=(1,1),stride=(1,1),out_channels=512), # 1x1x512 -> 1x1x512
            nn.ReLU(),
        )
        self.CBAM_module=CBAM(512)
        self.Conv=nn.Sequential(
            nn.Conv2d(in_channels=512*7*7+512,kernel_size=(1,1),stride=(1,1),out_channels=1024),
            nn.ReLU(),
            nn.Conv2d(in_channels=1024,kernel_size=(1,1),stride=(1,1),out_channels=512),
            nn.ReLU()
        )


    def forward(self,embedding1,compact_feature1,embedding2,compact_feature2):
        F1=torch.unsqueeze(torch.squeeze(self.extract_flatten_feature(compact_feature1)),2) # 1x1x512 -> 512
        F2=torch.unsqueeze(torch.squeeze(self.extract_flatten_feature(compact_feature2)),1) # 1x1x512 -> 512
        
        correlation_map=torch.exp(torch.matmul(F1,F2)/(512**0.5))  #(_,512,512)
        denominator1=torch.sum(correlation_map,dim=2) # sum of each col
        denominator2=torch.sum(correlation_map,dim=1) # sum of each row
        

        cross_attention_feature1=torch.matmul( correlation_map,compact_feature1.view(-1,512,7*7)) / torch.unsqueeze(denominator1,dim=2) # (_,256,) broadcast to each col -> 
        cross_attention_feature2=torch.matmul( torch.permute(correlation_map,(0,2,1)),compact_feature2.view(-1,512,7*7))/ torch.unsqueeze(denominator2,dim=2) # (_,512,7,1) broadcast to each col

        aggregate_feature1=self.CBAM_module(cross_attention_feature1.view(-1,512,7,7)+compact_feature1)
        aggregate_feature2=self.CBAM_module(cross_attention_feature2.view(-1,512,7,7)+compact_feature2)

        concate_feature1=(torch.cat([embedding1,aggregate_feature1],dim=1))[...,None,None]# 512*7*7+512
        concate_feature2=(torch.cat([embedding2,aggregate_feature2],dim=1))[...,None,None] # 512*7*7+512

        x_out1=torch.squeeze(self.Conv(concate_feature1))
        x_out2=torch.squeeze(self.Conv(concate_feature2))

        return torch.squeeze(concate_feature1),torch.squeeze(concate_feature2),x_out1,x_out2

class Head(nn.Module):
    def __init__(self,in_features=512,out_features=4,ratio=8):
        super().__init__()
        self.projection_head=nn.Sequential(
            torch.nn.Linear(in_features, in_features//ratio),
            torch.nn.BatchNorm1d(in_features//ratio),
            torch.nn.ReLU(),
            torch.nn.Linear(in_features//ratio, out_features),
        )
        
        self.initialize_weights(self.projection_head)
        
    def initialize_weights(self,proj_head):
        for m in proj_head.modules():
            if isinstance(m, nn.Linear):
                nn.init.uniform_(m.weight -0.05, 0.05)
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.BatchNorm1d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
    def forward(self,em):
        return self.projection_head(em)

class Debias(nn.Module):
    def __init__(self,in_feature,out_feature=512):
        super().__init__()
        self.in_feature=in_feature
        self.out_feature=out_feature
        self.Debias=nn.Linear(in_feature,out_feature)
    def forward(self,x1,x2):
        Blen=len(x1)*2
        x1x2=torch.cat([x1,x2],dim=0)
        x2x1=torch.cat([x2,x1],dim=0)
        
        middle=((torch.unsqueeze(x1x2,dim=1)+torch.unsqueeze(x1x2,dim=0))/2).view(-1,self.in_feature) # add by broadcast
        fmiddle=self.Debias(middle).view(Blen,Blen,-1)
        
        fx1x2=self.Debias(x1x2)
        fx2x1=self.Debias(x2x1)
        
        bias_map=torch.cosine_similarity(torch.unsqueeze(fx1x2,dim=1),fmiddle,dim=2)**2-torch.cosine_similarity(torch.unsqueeze(fx1x2,dim=0),fmiddle,dim=2)**2
        
        pair_mid=(x1x2+x2x1)/2
        fmix=self.Debias(pair_mid)
        bias_pair=torch.cosine_similarity(fx1x2,fmix,dim=1)**2-torch.cosine_similarity(fx2x1,fmix,dim=1)**2

        return bias_map,bias_pair
    
class MultiTask(nn.Module):
    def __init__(self):
        super().__init__()
        self.encoder=KitModel("./kit_resnet101.pkl")
        self.Atten_module=Atten()
        self.task_race=Head(512,4,8)
        self.debias_layer=Debias(512*7*7+512)
    def forward(self,imgs,alpha=1.0):
        img1,img2=imgs
        e1,compact_feature1=self.encoder(img1) # each featur 16, 512x7x7
        e2,compact_feature2=self.encoder(img2)
        atten_em1,atten_em2,x1,x2=self.Atten_module(e1,compact_feature1,e2,compact_feature2)
        reverse_em1,reverse_em2=grad_reverse(e1,1.),grad_reverse(e2,1.)
        r1,r2=self.task_race(reverse_em1),self.task_race(reverse_em2)
        biasmap,bias_pair=self.debias_layer(atten_em1,atten_em2)

        return r1,r2,e1,e2,x1,x2,biasmap,bias_pair
    
    @staticmethod
    def save_model(model,path):
        torch.save(model.state_dict(),path)
    def load(self,path,num=0):
        self.encoder.load_state_dict(torch.load(os.path.join(path,f'encoder{num}.pth'))) 
        self.Atten_module.load_state_dict(torch.load(os.path.join(path,f'Atten{num}.pth'))) 
        self.task_race.load_state_dict(torch.load(os.path.join(path,f'task_race{num}.pth')))
        self.debias_layer.load_state_dict(torch.load(os.path.join(path,f'debias_layer{num}.pth')))
    def save(self,path,num=0):
        self.save_model(self.encoder , os.path.join(path,f'encoder{num}.pth'))
        self.save_model(self.Atten_module , os.path.join(path,f'Atten{num}.pth'))
        self.save_model(self.task_race , os.path.join(path,f'task_race{num}.pth'))
        self.save_model(self.debias_layer , os.path.join(path,f'debias_layer{num}.pth'))
    
    

models_multi_task.py
garynlfd/KFC
import argparse
import numpy as np
import gc
from models_multi_task import *
import sys
import torch
from keras.preprocessing import image
import os
from utils import *
import heapq

def baseline_model(model_path):
    model = MultiTask().cuda()
    model.load(model_path, "_std_best")
    model.eval()
    return model

def get_test(sample_path,res):
    test_file_path = os.path.join(sample_path,"./aligned_dataset/mixed_dataset/mixed_dataset_test_cross.txt")
    test=[]
    f = open(test_file_path, "r+", encoding='utf-8')
    file_leng = 0
    while True:
        line=f.readline().replace('\n','')
        if not line:
            break
        else:
            test.append(line.split(' '))
            file_leng += 1
    f.close()
    res['avg'][0]=len(test)
    for now in test:
        res[now[2]][0]+=1
    return test, file_leng


def gen(list_tuples, batch_size):
    total=len(list_tuples)
    start=0
    while True:
        if start+batch_size<total:
            end=start+batch_size
        else:
            end=total
        batch_list=list_tuples[start:end]
        datas=[]
        labels=[]
        classes=[]
        races=[]
        id1=[]
        id2=[]

        for now in batch_list:
            datas.append([now[0],now[1]])
            labels.append(int(now[3]))
            classes.append(now[2])
            races.append(now[4])
            id1.append(0)

        X1 = np.array([read_image(x[0]) for x in datas])
        X2 = np.array([read_image(x[1]) for x in datas])
        yield datas, X1, X2, labels,classes,races,id1,id2,batch_list
        start=end
        if start == total:
            yield None,None,None,None,None,None,None,None,None
        gc.collect()


def read_image(path):
    img = image.load_img(path, target_size=(112, 112))
    img = np.array(img).astype(np.float)
    return np.transpose(img, (2, 0, 1))


def test(args):
    model_path = os.path.join(args.save_path,"weights_Cornell_multi")
    sample_path = args.sample
    batch_size = args.batch_size
    log_path = args.log_path
    threshold = args.threshold
    model = baseline_model(model_path)
    classes = [
        'fd', 'md', 'fs', 'ms', 'avg'
    ]
    res={}
    for n in classes:
        res[n]=[0,0]
    test_samples, file_leng = get_test(sample_path, res)
    easy_kin_similarities = [] # p=labels=1
    hard_kin_similarities = [] # p=0, labels=1
    easy_non_kin_similarities = [] # p=labels=0
    hard_non_kin_similarities = [] # p=1, labels=0
    with torch.no_grad():
        fnp = 0
        fpp = 0
        for datas, img1, img2, labels, classes, races, id1, id2, batch_list in gen(test_samples, batch_size):
            if img1 is not None:
                img1 = torch.from_numpy(img1).type(torch.float).cuda()
                img2 = torch.from_numpy(img2).type(torch.float).cuda()
                id1 = torch.Tensor(id1).type(torch.int).cuda()
                id2 = torch.Tensor(id2).type(torch.int).cuda()
                r1,r2,e1,e2,x1,x2,bias_map,bias_pair = model([img1, img2])
                pred = torch.cosine_similarity(e1, e2, dim=1).cpu().detach().numpy().tolist()
                for i in range(len(pred)):
                    if pred[i] >= threshold:
                        p = 1
                    else:
                        p = 0
                    if p == labels[i]:
                        res['avg'][1] += 1
                        res[classes[i]][1] += 1
                        if p == 1:
                            if len(easy_kin_similarities) < 60:
                                heapq.heappush(easy_kin_similarities, (pred[i], datas[i]))
                            else:
                                heapq.heappushpop(easy_kin_similarities, (pred[i], datas[i]))
                        else: 
                            if len(easy_non_kin_similarities) < 30:
                                heapq.heappush(easy_non_kin_similarities, (pred[i], datas[i]))
                            else:
                                heapq.heappushpop(easy_non_kin_similarities, (pred[i], datas[i]))
                    else:
                        neg_similarity = -pred[i]
                        if labels[i] == 1: # p = 0; label = 1 -> false negative pair
                            fnp = fnp + 1
                            if len(hard_kin_similarities) < 30:
                                heapq.heappush(hard_kin_similarities, (neg_similarity, datas[i]))
                            else:
                                heapq.heappushpop(hard_kin_similarities, (neg_similarity, datas[i]))
                        else: # p = 1; label = 0 -> false positive pair
                            fpp = fpp + 1
                            if len(hard_non_kin_similarities) < 30:
                                heapq.heappush(hard_non_kin_similarities, (neg_similarity, datas[i]))
                            else:
                                heapq.heappushpop(hard_non_kin_similarities, (neg_similarity, datas[i]))
            else:
                break
    mylog("number of false negative pairs: ", fnp, path=log_path)
    mylog("false negative rate: ", fnp / file_leng, path=log_path)
    mylog("number of false positive pairs: ", fpp, path=log_path)
    mylog("false positive rate: ", fpp / file_leng, path=log_path)
    for key in res:
        if res[key][0] == 0: continue
        mylog(key, ':', res[key][1] / res[key][0], path=log_path)


    print("easy KIN cosine similarities:")
    for similarity, datas in heapq.nlargest(60, easy_kin_similarities):
        print(f"Image Pair: {datas}, Similarity: {similarity}")
    
    print("-"*20)

    print("easy non-KIN cosine similarities:")
    for similarity, datas in heapq.nlargest(30, easy_non_kin_similarities):
        print(f"Image Pair: {datas}, Similarity: {similarity}")
    
    print("-"*20)

    print("hard KIN cosine similarities:")
    for similarity, datas in heapq.nlargest(30, hard_kin_similarities):
        print(f"Image Pair: {datas}, Similarity: {similarity}")

    print("-"*20)

    print("hard non-KIN cosine similarities:")
    for similarity, datas in heapq.nlargest(30, hard_non_kin_similarities):
        print(f"Image Pair: {datas}, Similarity: {similarity}")


if __name__=="__main__":
    parser = argparse.ArgumentParser(description="test  accuracy")
    parser.add_argument("--sample", type=str, help="sample root")
    parser.add_argument("--save_path", type=str, help="model save path")
    parser.add_argument("--threshold", type=float, help=" threshold ")
    parser.add_argument("--batch_size", type=int, default=40, help="batch size default 40")
    parser.add_argument("--log_path", type=str, default="./log.txt", help="log path default log.txt ")
    parser.add_argument("--gpu", default="1", type=str, help="gpu id you use")
    args = parser.parse_args()
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
    set_seed(100)
    test(args)

test.py
garynlfd/KFC
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import math
import pickle
# from Track1.utils import *
from utils import *
_weights_dict = dict()

def load_weights(weight_file):
    if weight_file == None:
        return

    try:
        weights_dict = np.load(weight_file, allow_pickle=True).item()
        # weights_dict = torch.load(weight_file)
    except:
        weights_dict = np.load(weight_file, allow_pickle=True, encoding='bytes').item()
        # weights_dict = torch.load(weight_file)

    return weights_dict

class BasicConv(nn.Module):
    def __init__(self, in_planes, out_planes, kernel_size, stride=1, padding=0, dilation=1, groups=1, relu=True, bn=True, bias=False):
        super(BasicConv, self).__init__()
        self.out_channels = out_planes
        self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias)
        self.bn = nn.BatchNorm2d(out_planes,eps=1e-5, momentum=0.01, affine=True) if bn else None
        self.relu = nn.ReLU() if relu else None

    def forward(self, x):
        x = self.conv(x)
        if self.bn is not None:
            x = self.bn(x)
        if self.relu is not None:
            x = self.relu(x)
        return x

class Flatten(torch.nn.Module):
    def forward(self, x):
        return x.view(x.size(0), -1)

class CI(torch.nn.Module):
    def __init__(self, gate_channels, flatten, reduction_ratio=16, pool_types=['avg', 'max']):
        super(CI, self).__init__()
        self.ChannelGate = ChannelGate(gate_channels)
        self.SpatialGate = SpatialGate(flatten)
    def forward(self, x):
        x_out = self.ChannelGate(x) # x_out size = 512, 7, 7
        x_out = self.SpatialGate(x_out) # x_out size = 25088(because flatten)

        return x_out

class ChannelGate(torch.nn.Module):
    def __init__(self,in_channel,ratio=16):
        super().__init__()
        self.CI_module=nn.Sequential(
            Flatten(),
            nn.Linear(in_channel, in_channel // 16),
            nn.ReLU(),
            nn.Linear(in_channel // 16, in_channel)
        )
    def forward(self,x):
        channel_att_sum = None
        avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))
        channel_att_avg = self.CI_module(avg_pool)
        max_pool = F.max_pool2d(x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))
        channel_att_max = self.CI_module(max_pool)
        channel_att_sum = channel_att_avg + channel_att_max
        scale = torch.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(x)
        x = x * scale
        return x
        # return nn.Flatten(x)

class ChannelPool(torch.nn.Module):
    def forward(self, x):
        return torch.cat((torch.max(x, 1)[0].unsqueeze(1), torch.mean(x, 1).unsqueeze(1)), dim=1)

class SpatialGate(nn.Module):
    def __init__(self, flatten):
        super(SpatialGate, self).__init__()
        kernel_size = 7
        self.flatten_bool = flatten
        self.compress = ChannelPool()
        self.spatial = BasicConv(2, 1, kernel_size, stride=1, padding=(kernel_size-1) // 2, relu=False)
        self.flatten=nn.Sequential(
            nn.Flatten()
        )
    def forward(self, x):
        x_compress = self.compress(x) # x_compress size = 2, 7, 7; 2 means 2 channel(max pool & avg pool)
        x_out = self.spatial(x_compress) # x_out size = 1, 7, 7
        scale = torch.sigmoid(x_out) # broadcasting
        x = x * scale
        # return x * scale
        if self.flatten_bool == True: # for CI
            return self.flatten(x)
        else: # for AttentionHead
            return x

class KitModel(nn.Module):

    
    def __init__(self, weight_file):
        super(KitModel, self).__init__()
        global _weights_dict
        _weights_dict = load_weights(weight_file)

        self.conv0 = self.__conv(2, name='conv0', in_channels=3, out_channels=64, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.bn0 = self.__batch_normalization(2, 'bn0', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit1_bn1 = self.__batch_normalization(2, 'stage1_unit1_bn1', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit1_conv1sc = self.__conv(2, name='stage1_unit1_conv1sc', in_channels=64, out_channels=64, kernel_size=(1, 1), stride=(2, 2), groups=1, bias=False)
        self.stage1_unit1_conv1 = self.__conv(2, name='stage1_unit1_conv1', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage1_unit1_sc = self.__batch_normalization(2, 'stage1_unit1_sc', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit1_bn2 = self.__batch_normalization(2, 'stage1_unit1_bn2', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit1_conv2 = self.__conv(2, name='stage1_unit1_conv2', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(2, 2), groups=1, bias=False)
        self.stage1_unit1_bn3 = self.__batch_normalization(2, 'stage1_unit1_bn3', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit2_bn1 = self.__batch_normalization(2, 'stage1_unit2_bn1', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit2_conv1 = self.__conv(2, name='stage1_unit2_conv1', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage1_unit2_bn2 = self.__batch_normalization(2, 'stage1_unit2_bn2', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit2_conv2 = self.__conv(2, name='stage1_unit2_conv2', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage1_unit2_bn3 = self.__batch_normalization(2, 'stage1_unit2_bn3', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit3_bn1 = self.__batch_normalization(2, 'stage1_unit3_bn1', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit3_conv1 = self.__conv(2, name='stage1_unit3_conv1', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage1_unit3_bn2 = self.__batch_normalization(2, 'stage1_unit3_bn2', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage1_unit3_conv2 = self.__conv(2, name='stage1_unit3_conv2', in_channels=64, out_channels=64, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage1_unit3_bn3 = self.__batch_normalization(2, 'stage1_unit3_bn3', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit1_bn1 = self.__batch_normalization(2, 'stage2_unit1_bn1', num_features=64, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit1_conv1sc = self.__conv(2, name='stage2_unit1_conv1sc', in_channels=64, out_channels=128, kernel_size=(1, 1), stride=(2, 2), groups=1, bias=False)
        self.stage2_unit1_conv1 = self.__conv(2, name='stage2_unit1_conv1', in_channels=64, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit1_sc = self.__batch_normalization(2, 'stage2_unit1_sc', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit1_bn2 = self.__batch_normalization(2, 'stage2_unit1_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit1_conv2 = self.__conv(2, name='stage2_unit1_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(2, 2), groups=1, bias=False)
        self.stage2_unit1_bn3 = self.__batch_normalization(2, 'stage2_unit1_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit2_bn1 = self.__batch_normalization(2, 'stage2_unit2_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit2_conv1 = self.__conv(2, name='stage2_unit2_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit2_bn2 = self.__batch_normalization(2, 'stage2_unit2_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit2_conv2 = self.__conv(2, name='stage2_unit2_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit2_bn3 = self.__batch_normalization(2, 'stage2_unit2_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit3_bn1 = self.__batch_normalization(2, 'stage2_unit3_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit3_conv1 = self.__conv(2, name='stage2_unit3_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit3_bn2 = self.__batch_normalization(2, 'stage2_unit3_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit3_conv2 = self.__conv(2, name='stage2_unit3_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit3_bn3 = self.__batch_normalization(2, 'stage2_unit3_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit4_bn1 = self.__batch_normalization(2, 'stage2_unit4_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit4_conv1 = self.__conv(2, name='stage2_unit4_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit4_bn2 = self.__batch_normalization(2, 'stage2_unit4_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit4_conv2 = self.__conv(2, name='stage2_unit4_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit4_bn3 = self.__batch_normalization(2, 'stage2_unit4_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit5_bn1 = self.__batch_normalization(2, 'stage2_unit5_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit5_conv1 = self.__conv(2, name='stage2_unit5_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit5_bn2 = self.__batch_normalization(2, 'stage2_unit5_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit5_conv2 = self.__conv(2, name='stage2_unit5_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit5_bn3 = self.__batch_normalization(2, 'stage2_unit5_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit6_bn1 = self.__batch_normalization(2, 'stage2_unit6_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit6_conv1 = self.__conv(2, name='stage2_unit6_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit6_bn2 = self.__batch_normalization(2, 'stage2_unit6_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit6_conv2 = self.__conv(2, name='stage2_unit6_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit6_bn3 = self.__batch_normalization(2, 'stage2_unit6_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit7_bn1 = self.__batch_normalization(2, 'stage2_unit7_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit7_conv1 = self.__conv(2, name='stage2_unit7_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit7_bn2 = self.__batch_normalization(2, 'stage2_unit7_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit7_conv2 = self.__conv(2, name='stage2_unit7_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit7_bn3 = self.__batch_normalization(2, 'stage2_unit7_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit8_bn1 = self.__batch_normalization(2, 'stage2_unit8_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit8_conv1 = self.__conv(2, name='stage2_unit8_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit8_bn2 = self.__batch_normalization(2, 'stage2_unit8_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit8_conv2 = self.__conv(2, name='stage2_unit8_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit8_bn3 = self.__batch_normalization(2, 'stage2_unit8_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit9_bn1 = self.__batch_normalization(2, 'stage2_unit9_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit9_conv1 = self.__conv(2, name='stage2_unit9_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit9_bn2 = self.__batch_normalization(2, 'stage2_unit9_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit9_conv2 = self.__conv(2, name='stage2_unit9_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit9_bn3 = self.__batch_normalization(2, 'stage2_unit9_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit10_bn1 = self.__batch_normalization(2, 'stage2_unit10_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit10_conv1 = self.__conv(2, name='stage2_unit10_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit10_bn2 = self.__batch_normalization(2, 'stage2_unit10_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit10_conv2 = self.__conv(2, name='stage2_unit10_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit10_bn3 = self.__batch_normalization(2, 'stage2_unit10_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit11_bn1 = self.__batch_normalization(2, 'stage2_unit11_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit11_conv1 = self.__conv(2, name='stage2_unit11_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit11_bn2 = self.__batch_normalization(2, 'stage2_unit11_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit11_conv2 = self.__conv(2, name='stage2_unit11_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit11_bn3 = self.__batch_normalization(2, 'stage2_unit11_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit12_bn1 = self.__batch_normalization(2, 'stage2_unit12_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit12_conv1 = self.__conv(2, name='stage2_unit12_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit12_bn2 = self.__batch_normalization(2, 'stage2_unit12_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit12_conv2 = self.__conv(2, name='stage2_unit12_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit12_bn3 = self.__batch_normalization(2, 'stage2_unit12_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit13_bn1 = self.__batch_normalization(2, 'stage2_unit13_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit13_conv1 = self.__conv(2, name='stage2_unit13_conv1', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit13_bn2 = self.__batch_normalization(2, 'stage2_unit13_bn2', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage2_unit13_conv2 = self.__conv(2, name='stage2_unit13_conv2', in_channels=128, out_channels=128, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage2_unit13_bn3 = self.__batch_normalization(2, 'stage2_unit13_bn3', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit1_bn1 = self.__batch_normalization(2, 'stage3_unit1_bn1', num_features=128, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit1_conv1sc = self.__conv(2, name='stage3_unit1_conv1sc', in_channels=128, out_channels=256, kernel_size=(1, 1), stride=(2, 2), groups=1, bias=False)
        self.stage3_unit1_conv1 = self.__conv(2, name='stage3_unit1_conv1', in_channels=128, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit1_sc = self.__batch_normalization(2, 'stage3_unit1_sc', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit1_bn2 = self.__batch_normalization(2, 'stage3_unit1_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit1_conv2 = self.__conv(2, name='stage3_unit1_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(2, 2), groups=1, bias=False)
        self.stage3_unit1_bn3 = self.__batch_normalization(2, 'stage3_unit1_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit2_bn1 = self.__batch_normalization(2, 'stage3_unit2_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit2_conv1 = self.__conv(2, name='stage3_unit2_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit2_bn2 = self.__batch_normalization(2, 'stage3_unit2_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit2_conv2 = self.__conv(2, name='stage3_unit2_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit2_bn3 = self.__batch_normalization(2, 'stage3_unit2_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit3_bn1 = self.__batch_normalization(2, 'stage3_unit3_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit3_conv1 = self.__conv(2, name='stage3_unit3_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit3_bn2 = self.__batch_normalization(2, 'stage3_unit3_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit3_conv2 = self.__conv(2, name='stage3_unit3_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit3_bn3 = self.__batch_normalization(2, 'stage3_unit3_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit4_bn1 = self.__batch_normalization(2, 'stage3_unit4_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit4_conv1 = self.__conv(2, name='stage3_unit4_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit4_bn2 = self.__batch_normalization(2, 'stage3_unit4_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit4_conv2 = self.__conv(2, name='stage3_unit4_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit4_bn3 = self.__batch_normalization(2, 'stage3_unit4_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit5_bn1 = self.__batch_normalization(2, 'stage3_unit5_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit5_conv1 = self.__conv(2, name='stage3_unit5_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit5_bn2 = self.__batch_normalization(2, 'stage3_unit5_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit5_conv2 = self.__conv(2, name='stage3_unit5_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit5_bn3 = self.__batch_normalization(2, 'stage3_unit5_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit6_bn1 = self.__batch_normalization(2, 'stage3_unit6_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit6_conv1 = self.__conv(2, name='stage3_unit6_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit6_bn2 = self.__batch_normalization(2, 'stage3_unit6_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit6_conv2 = self.__conv(2, name='stage3_unit6_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit6_bn3 = self.__batch_normalization(2, 'stage3_unit6_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit7_bn1 = self.__batch_normalization(2, 'stage3_unit7_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit7_conv1 = self.__conv(2, name='stage3_unit7_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit7_bn2 = self.__batch_normalization(2, 'stage3_unit7_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit7_conv2 = self.__conv(2, name='stage3_unit7_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit7_bn3 = self.__batch_normalization(2, 'stage3_unit7_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit8_bn1 = self.__batch_normalization(2, 'stage3_unit8_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit8_conv1 = self.__conv(2, name='stage3_unit8_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit8_bn2 = self.__batch_normalization(2, 'stage3_unit8_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit8_conv2 = self.__conv(2, name='stage3_unit8_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit8_bn3 = self.__batch_normalization(2, 'stage3_unit8_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit9_bn1 = self.__batch_normalization(2, 'stage3_unit9_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit9_conv1 = self.__conv(2, name='stage3_unit9_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit9_bn2 = self.__batch_normalization(2, 'stage3_unit9_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit9_conv2 = self.__conv(2, name='stage3_unit9_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit9_bn3 = self.__batch_normalization(2, 'stage3_unit9_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit10_bn1 = self.__batch_normalization(2, 'stage3_unit10_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit10_conv1 = self.__conv(2, name='stage3_unit10_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit10_bn2 = self.__batch_normalization(2, 'stage3_unit10_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit10_conv2 = self.__conv(2, name='stage3_unit10_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit10_bn3 = self.__batch_normalization(2, 'stage3_unit10_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit11_bn1 = self.__batch_normalization(2, 'stage3_unit11_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit11_conv1 = self.__conv(2, name='stage3_unit11_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit11_bn2 = self.__batch_normalization(2, 'stage3_unit11_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit11_conv2 = self.__conv(2, name='stage3_unit11_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit11_bn3 = self.__batch_normalization(2, 'stage3_unit11_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit12_bn1 = self.__batch_normalization(2, 'stage3_unit12_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit12_conv1 = self.__conv(2, name='stage3_unit12_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit12_bn2 = self.__batch_normalization(2, 'stage3_unit12_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit12_conv2 = self.__conv(2, name='stage3_unit12_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit12_bn3 = self.__batch_normalization(2, 'stage3_unit12_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit13_bn1 = self.__batch_normalization(2, 'stage3_unit13_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit13_conv1 = self.__conv(2, name='stage3_unit13_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit13_bn2 = self.__batch_normalization(2, 'stage3_unit13_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit13_conv2 = self.__conv(2, name='stage3_unit13_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit13_bn3 = self.__batch_normalization(2, 'stage3_unit13_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit14_bn1 = self.__batch_normalization(2, 'stage3_unit14_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit14_conv1 = self.__conv(2, name='stage3_unit14_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit14_bn2 = self.__batch_normalization(2, 'stage3_unit14_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit14_conv2 = self.__conv(2, name='stage3_unit14_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit14_bn3 = self.__batch_normalization(2, 'stage3_unit14_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit15_bn1 = self.__batch_normalization(2, 'stage3_unit15_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit15_conv1 = self.__conv(2, name='stage3_unit15_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit15_bn2 = self.__batch_normalization(2, 'stage3_unit15_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit15_conv2 = self.__conv(2, name='stage3_unit15_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit15_bn3 = self.__batch_normalization(2, 'stage3_unit15_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit16_bn1 = self.__batch_normalization(2, 'stage3_unit16_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit16_conv1 = self.__conv(2, name='stage3_unit16_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit16_bn2 = self.__batch_normalization(2, 'stage3_unit16_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit16_conv2 = self.__conv(2, name='stage3_unit16_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit16_bn3 = self.__batch_normalization(2, 'stage3_unit16_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit17_bn1 = self.__batch_normalization(2, 'stage3_unit17_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit17_conv1 = self.__conv(2, name='stage3_unit17_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit17_bn2 = self.__batch_normalization(2, 'stage3_unit17_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit17_conv2 = self.__conv(2, name='stage3_unit17_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit17_bn3 = self.__batch_normalization(2, 'stage3_unit17_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit18_bn1 = self.__batch_normalization(2, 'stage3_unit18_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit18_conv1 = self.__conv(2, name='stage3_unit18_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit18_bn2 = self.__batch_normalization(2, 'stage3_unit18_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit18_conv2 = self.__conv(2, name='stage3_unit18_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit18_bn3 = self.__batch_normalization(2, 'stage3_unit18_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit19_bn1 = self.__batch_normalization(2, 'stage3_unit19_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit19_conv1 = self.__conv(2, name='stage3_unit19_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit19_bn2 = self.__batch_normalization(2, 'stage3_unit19_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit19_conv2 = self.__conv(2, name='stage3_unit19_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit19_bn3 = self.__batch_normalization(2, 'stage3_unit19_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit20_bn1 = self.__batch_normalization(2, 'stage3_unit20_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit20_conv1 = self.__conv(2, name='stage3_unit20_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit20_bn2 = self.__batch_normalization(2, 'stage3_unit20_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit20_conv2 = self.__conv(2, name='stage3_unit20_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit20_bn3 = self.__batch_normalization(2, 'stage3_unit20_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit21_bn1 = self.__batch_normalization(2, 'stage3_unit21_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit21_conv1 = self.__conv(2, name='stage3_unit21_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit21_bn2 = self.__batch_normalization(2, 'stage3_unit21_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit21_conv2 = self.__conv(2, name='stage3_unit21_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit21_bn3 = self.__batch_normalization(2, 'stage3_unit21_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit22_bn1 = self.__batch_normalization(2, 'stage3_unit22_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit22_conv1 = self.__conv(2, name='stage3_unit22_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit22_bn2 = self.__batch_normalization(2, 'stage3_unit22_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit22_conv2 = self.__conv(2, name='stage3_unit22_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit22_bn3 = self.__batch_normalization(2, 'stage3_unit22_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit23_bn1 = self.__batch_normalization(2, 'stage3_unit23_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit23_conv1 = self.__conv(2, name='stage3_unit23_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit23_bn2 = self.__batch_normalization(2, 'stage3_unit23_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit23_conv2 = self.__conv(2, name='stage3_unit23_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit23_bn3 = self.__batch_normalization(2, 'stage3_unit23_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit24_bn1 = self.__batch_normalization(2, 'stage3_unit24_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit24_conv1 = self.__conv(2, name='stage3_unit24_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit24_bn2 = self.__batch_normalization(2, 'stage3_unit24_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit24_conv2 = self.__conv(2, name='stage3_unit24_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit24_bn3 = self.__batch_normalization(2, 'stage3_unit24_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit25_bn1 = self.__batch_normalization(2, 'stage3_unit25_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit25_conv1 = self.__conv(2, name='stage3_unit25_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit25_bn2 = self.__batch_normalization(2, 'stage3_unit25_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit25_conv2 = self.__conv(2, name='stage3_unit25_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit25_bn3 = self.__batch_normalization(2, 'stage3_unit25_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit26_bn1 = self.__batch_normalization(2, 'stage3_unit26_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit26_conv1 = self.__conv(2, name='stage3_unit26_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit26_bn2 = self.__batch_normalization(2, 'stage3_unit26_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit26_conv2 = self.__conv(2, name='stage3_unit26_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit26_bn3 = self.__batch_normalization(2, 'stage3_unit26_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit27_bn1 = self.__batch_normalization(2, 'stage3_unit27_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit27_conv1 = self.__conv(2, name='stage3_unit27_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit27_bn2 = self.__batch_normalization(2, 'stage3_unit27_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit27_conv2 = self.__conv(2, name='stage3_unit27_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit27_bn3 = self.__batch_normalization(2, 'stage3_unit27_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit28_bn1 = self.__batch_normalization(2, 'stage3_unit28_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit28_conv1 = self.__conv(2, name='stage3_unit28_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit28_bn2 = self.__batch_normalization(2, 'stage3_unit28_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit28_conv2 = self.__conv(2, name='stage3_unit28_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit28_bn3 = self.__batch_normalization(2, 'stage3_unit28_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit29_bn1 = self.__batch_normalization(2, 'stage3_unit29_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit29_conv1 = self.__conv(2, name='stage3_unit29_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit29_bn2 = self.__batch_normalization(2, 'stage3_unit29_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit29_conv2 = self.__conv(2, name='stage3_unit29_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit29_bn3 = self.__batch_normalization(2, 'stage3_unit29_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit30_bn1 = self.__batch_normalization(2, 'stage3_unit30_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit30_conv1 = self.__conv(2, name='stage3_unit30_conv1', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit30_bn2 = self.__batch_normalization(2, 'stage3_unit30_bn2', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage3_unit30_conv2 = self.__conv(2, name='stage3_unit30_conv2', in_channels=256, out_channels=256, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage3_unit30_bn3 = self.__batch_normalization(2, 'stage3_unit30_bn3', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit1_bn1 = self.__batch_normalization(2, 'stage4_unit1_bn1', num_features=256, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit1_conv1sc = self.__conv(2, name='stage4_unit1_conv1sc', in_channels=256, out_channels=512, kernel_size=(1, 1), stride=(2, 2), groups=1, bias=False)
        self.stage4_unit1_conv1 = self.__conv(2, name='stage4_unit1_conv1', in_channels=256, out_channels=512, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage4_unit1_sc = self.__batch_normalization(2, 'stage4_unit1_sc', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit1_bn2 = self.__batch_normalization(2, 'stage4_unit1_bn2', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit1_conv2 = self.__conv(2, name='stage4_unit1_conv2', in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(2, 2), groups=1, bias=False)
        self.stage4_unit1_bn3 = self.__batch_normalization(2, 'stage4_unit1_bn3', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit2_bn1 = self.__batch_normalization(2, 'stage4_unit2_bn1', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit2_conv1 = self.__conv(2, name='stage4_unit2_conv1', in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage4_unit2_bn2 = self.__batch_normalization(2, 'stage4_unit2_bn2', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit2_conv2 = self.__conv(2, name='stage4_unit2_conv2', in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage4_unit2_bn3 = self.__batch_normalization(2, 'stage4_unit2_bn3', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit3_bn1 = self.__batch_normalization(2, 'stage4_unit3_bn1', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit3_conv1 = self.__conv(2, name='stage4_unit3_conv1', in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage4_unit3_bn2 = self.__batch_normalization(2, 'stage4_unit3_bn2', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.stage4_unit3_conv2 = self.__conv(2, name='stage4_unit3_conv2', in_channels=512, out_channels=512, kernel_size=(3, 3), stride=(1, 1), groups=1, bias=False)
        self.stage4_unit3_bn3 = self.__batch_normalization(2, 'stage4_unit3_bn3', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.bn1 = self.__batch_normalization(2, 'bn1', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.pre_fc1 = self.__dense(name = 'pre_fc1', in_features = 25088, out_features = 512, bias = True)
        self.fc1 = self.__batch_normalization(0, 'fc1', num_features=512, eps=1.9999999494757503e-05, momentum=0.8999999761581421)
        self.CI_46 = CI(512, False)
        self.CI_47 = CI(512, False)
        self.CI_48 = CI(512, False)


    def forward(self, x):
        self.minusscalar0_second = torch.autograd.Variable(
            np2tensor(_weights_dict['minusscalar0_second']['value']), requires_grad=False)
        self.mulscalar0_second = torch.autograd.Variable(np2tensor(_weights_dict['mulscalar0_second']['value']),
                                                         requires_grad=False)
        minusscalar0    = x - self.minusscalar0_second
        mulscalar0      = minusscalar0 * self.mulscalar0_second
        conv0_pad       = F.pad(mulscalar0, (1, 1, 1, 1))
        conv0           = self.conv0(conv0_pad)
        bn0             = self.bn0(conv0)
        relu0           = F.prelu(bn0, np2tensor(_weights_dict['relu0']['weights']))
        stage1_unit1_bn1 = self.stage1_unit1_bn1(relu0)
        stage1_unit1_conv1sc = self.stage1_unit1_conv1sc(relu0)
        stage1_unit1_conv1_pad = F.pad(stage1_unit1_bn1, (1, 1, 1, 1))
        stage1_unit1_conv1 = self.stage1_unit1_conv1(stage1_unit1_conv1_pad)
        stage1_unit1_sc = self.stage1_unit1_sc(stage1_unit1_conv1sc)
        stage1_unit1_bn2 = self.stage1_unit1_bn2(stage1_unit1_conv1)
        stage1_unit1_relu1 = F.prelu(stage1_unit1_bn2, np2tensor(_weights_dict['stage1_unit1_relu1']['weights']))
        stage1_unit1_conv2_pad = F.pad(stage1_unit1_relu1, (1, 1, 1, 1))
        stage1_unit1_conv2 = self.stage1_unit1_conv2(stage1_unit1_conv2_pad)
        stage1_unit1_bn3 = self.stage1_unit1_bn3(stage1_unit1_conv2)
        plus0           = stage1_unit1_bn3 + stage1_unit1_sc
        stage1_unit2_bn1 = self.stage1_unit2_bn1(plus0)
        stage1_unit2_conv1_pad = F.pad(stage1_unit2_bn1, (1, 1, 1, 1))
        stage1_unit2_conv1 = self.stage1_unit2_conv1(stage1_unit2_conv1_pad)
        stage1_unit2_bn2 = self.stage1_unit2_bn2(stage1_unit2_conv1)
        stage1_unit2_relu1 = F.prelu(stage1_unit2_bn2, np2tensor(_weights_dict['stage1_unit2_relu1']['weights']))
        stage1_unit2_conv2_pad = F.pad(stage1_unit2_relu1, (1, 1, 1, 1))
        stage1_unit2_conv2 = self.stage1_unit2_conv2(stage1_unit2_conv2_pad)
        stage1_unit2_bn3 = self.stage1_unit2_bn3(stage1_unit2_conv2)
        plus1           = stage1_unit2_bn3 + plus0
        stage1_unit3_bn1 = self.stage1_unit3_bn1(plus1)
        stage1_unit3_conv1_pad = F.pad(stage1_unit3_bn1, (1, 1, 1, 1))
        stage1_unit3_conv1 = self.stage1_unit3_conv1(stage1_unit3_conv1_pad)
        stage1_unit3_bn2 = self.stage1_unit3_bn2(stage1_unit3_conv1)
        stage1_unit3_relu1 = F.prelu(stage1_unit3_bn2, np2tensor(_weights_dict['stage1_unit3_relu1']['weights']))
        stage1_unit3_conv2_pad = F.pad(stage1_unit3_relu1, (1, 1, 1, 1))
        stage1_unit3_conv2 = self.stage1_unit3_conv2(stage1_unit3_conv2_pad)
        stage1_unit3_bn3 = self.stage1_unit3_bn3(stage1_unit3_conv2)
        plus2           = stage1_unit3_bn3 + plus1
        stage2_unit1_bn1 = self.stage2_unit1_bn1(plus2)
        stage2_unit1_conv1sc = self.stage2_unit1_conv1sc(plus2)
        stage2_unit1_conv1_pad = F.pad(stage2_unit1_bn1, (1, 1, 1, 1))
        stage2_unit1_conv1 = self.stage2_unit1_conv1(stage2_unit1_conv1_pad)
        stage2_unit1_sc = self.stage2_unit1_sc(stage2_unit1_conv1sc)
        stage2_unit1_bn2 = self.stage2_unit1_bn2(stage2_unit1_conv1)
        stage2_unit1_relu1 = F.prelu(stage2_unit1_bn2, np2tensor(_weights_dict['stage2_unit1_relu1']['weights']))
        stage2_unit1_conv2_pad = F.pad(stage2_unit1_relu1, (1, 1, 1, 1))
        stage2_unit1_conv2 = self.stage2_unit1_conv2(stage2_unit1_conv2_pad)
        stage2_unit1_bn3 = self.stage2_unit1_bn3(stage2_unit1_conv2)
        plus3           = stage2_unit1_bn3 + stage2_unit1_sc
        stage2_unit2_bn1 = self.stage2_unit2_bn1(plus3)
        stage2_unit2_conv1_pad = F.pad(stage2_unit2_bn1, (1, 1, 1, 1))
        stage2_unit2_conv1 = self.stage2_unit2_conv1(stage2_unit2_conv1_pad)
        stage2_unit2_bn2 = self.stage2_unit2_bn2(stage2_unit2_conv1)
        stage2_unit2_relu1 = F.prelu(stage2_unit2_bn2, np2tensor(_weights_dict['stage2_unit2_relu1']['weights']))
        stage2_unit2_conv2_pad = F.pad(stage2_unit2_relu1, (1, 1, 1, 1))
        stage2_unit2_conv2 = self.stage2_unit2_conv2(stage2_unit2_conv2_pad)
        stage2_unit2_bn3 = self.stage2_unit2_bn3(stage2_unit2_conv2)
        plus4           = stage2_unit2_bn3 + plus3
        stage2_unit3_bn1 = self.stage2_unit3_bn1(plus4)
        stage2_unit3_conv1_pad = F.pad(stage2_unit3_bn1, (1, 1, 1, 1))
        stage2_unit3_conv1 = self.stage2_unit3_conv1(stage2_unit3_conv1_pad)
        stage2_unit3_bn2 = self.stage2_unit3_bn2(stage2_unit3_conv1)
        stage2_unit3_relu1 = F.prelu(stage2_unit3_bn2, np2tensor(_weights_dict['stage2_unit3_relu1']['weights']))
        stage2_unit3_conv2_pad = F.pad(stage2_unit3_relu1, (1, 1, 1, 1))
        stage2_unit3_conv2 = self.stage2_unit3_conv2(stage2_unit3_conv2_pad)
        stage2_unit3_bn3 = self.stage2_unit3_bn3(stage2_unit3_conv2)
        plus5           = stage2_unit3_bn3 + plus4
        stage2_unit4_bn1 = self.stage2_unit4_bn1(plus5)
        stage2_unit4_conv1_pad = F.pad(stage2_unit4_bn1, (1, 1, 1, 1))
        stage2_unit4_conv1 = self.stage2_unit4_conv1(stage2_unit4_conv1_pad)
        stage2_unit4_bn2 = self.stage2_unit4_bn2(stage2_unit4_conv1)
        stage2_unit4_relu1 = F.prelu(stage2_unit4_bn2, np2tensor(_weights_dict['stage2_unit4_relu1']['weights']))
        stage2_unit4_conv2_pad = F.pad(stage2_unit4_relu1, (1, 1, 1, 1))
        stage2_unit4_conv2 = self.stage2_unit4_conv2(stage2_unit4_conv2_pad)
        stage2_unit4_bn3 = self.stage2_unit4_bn3(stage2_unit4_conv2)
        plus6           = stage2_unit4_bn3 + plus5
        stage2_unit5_bn1 = self.stage2_unit5_bn1(plus6)
        stage2_unit5_conv1_pad = F.pad(stage2_unit5_bn1, (1, 1, 1, 1))
        stage2_unit5_conv1 = self.stage2_unit5_conv1(stage2_unit5_conv1_pad)
        stage2_unit5_bn2 = self.stage2_unit5_bn2(stage2_unit5_conv1)
        stage2_unit5_relu1 = F.prelu(stage2_unit5_bn2, np2tensor(_weights_dict['stage2_unit5_relu1']['weights']))
        stage2_unit5_conv2_pad = F.pad(stage2_unit5_relu1, (1, 1, 1, 1))
        stage2_unit5_conv2 = self.stage2_unit5_conv2(stage2_unit5_conv2_pad)
        stage2_unit5_bn3 = self.stage2_unit5_bn3(stage2_unit5_conv2)
        plus7           = stage2_unit5_bn3 + plus6
        stage2_unit6_bn1 = self.stage2_unit6_bn1(plus7)
        stage2_unit6_conv1_pad = F.pad(stage2_unit6_bn1, (1, 1, 1, 1))
        stage2_unit6_conv1 = self.stage2_unit6_conv1(stage2_unit6_conv1_pad)
        stage2_unit6_bn2 = self.stage2_unit6_bn2(stage2_unit6_conv1)
        stage2_unit6_relu1 = F.prelu(stage2_unit6_bn2, np2tensor(_weights_dict['stage2_unit6_relu1']['weights']))
        stage2_unit6_conv2_pad = F.pad(stage2_unit6_relu1, (1, 1, 1, 1))
        stage2_unit6_conv2 = self.stage2_unit6_conv2(stage2_unit6_conv2_pad)
        stage2_unit6_bn3 = self.stage2_unit6_bn3(stage2_unit6_conv2)
        plus8           = stage2_unit6_bn3 + plus7
        stage2_unit7_bn1 = self.stage2_unit7_bn1(plus8)
        stage2_unit7_conv1_pad = F.pad(stage2_unit7_bn1, (1, 1, 1, 1))
        stage2_unit7_conv1 = self.stage2_unit7_conv1(stage2_unit7_conv1_pad)
        stage2_unit7_bn2 = self.stage2_unit7_bn2(stage2_unit7_conv1)
        stage2_unit7_relu1 = F.prelu(stage2_unit7_bn2, np2tensor(_weights_dict['stage2_unit7_relu1']['weights']))
        stage2_unit7_conv2_pad = F.pad(stage2_unit7_relu1, (1, 1, 1, 1))
        stage2_unit7_conv2 = self.stage2_unit7_conv2(stage2_unit7_conv2_pad)
        stage2_unit7_bn3 = self.stage2_unit7_bn3(stage2_unit7_conv2)
        plus9           = stage2_unit7_bn3 + plus8
        stage2_unit8_bn1 = self.stage2_unit8_bn1(plus9)
        stage2_unit8_conv1_pad = F.pad(stage2_unit8_bn1, (1, 1, 1, 1))
        stage2_unit8_conv1 = self.stage2_unit8_conv1(stage2_unit8_conv1_pad)
        stage2_unit8_bn2 = self.stage2_unit8_bn2(stage2_unit8_conv1)
        stage2_unit8_relu1 = F.prelu(stage2_unit8_bn2, np2tensor(_weights_dict['stage2_unit8_relu1']['weights']))
        stage2_unit8_conv2_pad = F.pad(stage2_unit8_relu1, (1, 1, 1, 1))
        stage2_unit8_conv2 = self.stage2_unit8_conv2(stage2_unit8_conv2_pad)
        stage2_unit8_bn3 = self.stage2_unit8_bn3(stage2_unit8_conv2)
        plus10          = stage2_unit8_bn3 + plus9
        stage2_unit9_bn1 = self.stage2_unit9_bn1(plus10)
        stage2_unit9_conv1_pad = F.pad(stage2_unit9_bn1, (1, 1, 1, 1))
        stage2_unit9_conv1 = self.stage2_unit9_conv1(stage2_unit9_conv1_pad)
        stage2_unit9_bn2 = self.stage2_unit9_bn2(stage2_unit9_conv1)
        stage2_unit9_relu1 = F.prelu(stage2_unit9_bn2, np2tensor(_weights_dict['stage2_unit9_relu1']['weights']))
        stage2_unit9_conv2_pad = F.pad(stage2_unit9_relu1, (1, 1, 1, 1))
        stage2_unit9_conv2 = self.stage2_unit9_conv2(stage2_unit9_conv2_pad)
        stage2_unit9_bn3 = self.stage2_unit9_bn3(stage2_unit9_conv2)
        plus11          = stage2_unit9_bn3 + plus10
        stage2_unit10_bn1 = self.stage2_unit10_bn1(plus11)
        stage2_unit10_conv1_pad = F.pad(stage2_unit10_bn1, (1, 1, 1, 1))
        stage2_unit10_conv1 = self.stage2_unit10_conv1(stage2_unit10_conv1_pad)
        stage2_unit10_bn2 = self.stage2_unit10_bn2(stage2_unit10_conv1)
        stage2_unit10_relu1 = F.prelu(stage2_unit10_bn2, np2tensor(_weights_dict['stage2_unit10_relu1']['weights']))
        stage2_unit10_conv2_pad = F.pad(stage2_unit10_relu1, (1, 1, 1, 1))
        stage2_unit10_conv2 = self.stage2_unit10_conv2(stage2_unit10_conv2_pad)
        stage2_unit10_bn3 = self.stage2_unit10_bn3(stage2_unit10_conv2)
        plus12          = stage2_unit10_bn3 + plus11
        stage2_unit11_bn1 = self.stage2_unit11_bn1(plus12)
        stage2_unit11_conv1_pad = F.pad(stage2_unit11_bn1, (1, 1, 1, 1))
        stage2_unit11_conv1 = self.stage2_unit11_conv1(stage2_unit11_conv1_pad)
        stage2_unit11_bn2 = self.stage2_unit11_bn2(stage2_unit11_conv1)
        stage2_unit11_relu1 = F.prelu(stage2_unit11_bn2, np2tensor(_weights_dict['stage2_unit11_relu1']['weights']))
        stage2_unit11_conv2_pad = F.pad(stage2_unit11_relu1, (1, 1, 1, 1))
        stage2_unit11_conv2 = self.stage2_unit11_conv2(stage2_unit11_conv2_pad)
        stage2_unit11_bn3 = self.stage2_unit11_bn3(stage2_unit11_conv2)
        plus13          = stage2_unit11_bn3 + plus12
        stage2_unit12_bn1 = self.stage2_unit12_bn1(plus13)
        stage2_unit12_conv1_pad = F.pad(stage2_unit12_bn1, (1, 1, 1, 1))
        stage2_unit12_conv1 = self.stage2_unit12_conv1(stage2_unit12_conv1_pad)
        stage2_unit12_bn2 = self.stage2_unit12_bn2(stage2_unit12_conv1)
        stage2_unit12_relu1 = F.prelu(stage2_unit12_bn2, np2tensor(_weights_dict['stage2_unit12_relu1']['weights']))
        stage2_unit12_conv2_pad = F.pad(stage2_unit12_relu1, (1, 1, 1, 1))
        stage2_unit12_conv2 = self.stage2_unit12_conv2(stage2_unit12_conv2_pad)
        stage2_unit12_bn3 = self.stage2_unit12_bn3(stage2_unit12_conv2)
        plus14          = stage2_unit12_bn3 + plus13
        stage2_unit13_bn1 = self.stage2_unit13_bn1(plus14)
        stage2_unit13_conv1_pad = F.pad(stage2_unit13_bn1, (1, 1, 1, 1))
        stage2_unit13_conv1 = self.stage2_unit13_conv1(stage2_unit13_conv1_pad)
        stage2_unit13_bn2 = self.stage2_unit13_bn2(stage2_unit13_conv1)
        stage2_unit13_relu1 = F.prelu(stage2_unit13_bn2, np2tensor(_weights_dict['stage2_unit13_relu1']['weights']))
        stage2_unit13_conv2_pad = F.pad(stage2_unit13_relu1, (1, 1, 1, 1))
        stage2_unit13_conv2 = self.stage2_unit13_conv2(stage2_unit13_conv2_pad)
        stage2_unit13_bn3 = self.stage2_unit13_bn3(stage2_unit13_conv2)
        plus15          = stage2_unit13_bn3 + plus14
        stage3_unit1_bn1 = self.stage3_unit1_bn1(plus15)
        stage3_unit1_conv1sc = self.stage3_unit1_conv1sc(plus15)
        stage3_unit1_conv1_pad = F.pad(stage3_unit1_bn1, (1, 1, 1, 1))
        stage3_unit1_conv1 = self.stage3_unit1_conv1(stage3_unit1_conv1_pad)
        stage3_unit1_sc = self.stage3_unit1_sc(stage3_unit1_conv1sc)
        stage3_unit1_bn2 = self.stage3_unit1_bn2(stage3_unit1_conv1)
        stage3_unit1_relu1 = F.prelu(stage3_unit1_bn2, np2tensor(_weights_dict['stage3_unit1_relu1']['weights']))
        stage3_unit1_conv2_pad = F.pad(stage3_unit1_relu1, (1, 1, 1, 1))
        stage3_unit1_conv2 = self.stage3_unit1_conv2(stage3_unit1_conv2_pad)
        stage3_unit1_bn3 = self.stage3_unit1_bn3(stage3_unit1_conv2)
        plus16          = stage3_unit1_bn3 + stage3_unit1_sc
        stage3_unit2_bn1 = self.stage3_unit2_bn1(plus16)
        stage3_unit2_conv1_pad = F.pad(stage3_unit2_bn1, (1, 1, 1, 1))
        stage3_unit2_conv1 = self.stage3_unit2_conv1(stage3_unit2_conv1_pad)
        stage3_unit2_bn2 = self.stage3_unit2_bn2(stage3_unit2_conv1)
        stage3_unit2_relu1 = F.prelu(stage3_unit2_bn2, np2tensor(_weights_dict['stage3_unit2_relu1']['weights']))
        stage3_unit2_conv2_pad = F.pad(stage3_unit2_relu1, (1, 1, 1, 1))
        stage3_unit2_conv2 = self.stage3_unit2_conv2(stage3_unit2_conv2_pad)
        stage3_unit2_bn3 = self.stage3_unit2_bn3(stage3_unit2_conv2)
        plus17          = stage3_unit2_bn3 + plus16
        stage3_unit3_bn1 = self.stage3_unit3_bn1(plus17)
        stage3_unit3_conv1_pad = F.pad(stage3_unit3_bn1, (1, 1, 1, 1))
        stage3_unit3_conv1 = self.stage3_unit3_conv1(stage3_unit3_conv1_pad)
        stage3_unit3_bn2 = self.stage3_unit3_bn2(stage3_unit3_conv1)
        stage3_unit3_relu1 = F.prelu(stage3_unit3_bn2, np2tensor(_weights_dict['stage3_unit3_relu1']['weights']))
        stage3_unit3_conv2_pad = F.pad(stage3_unit3_relu1, (1, 1, 1, 1))
        stage3_unit3_conv2 = self.stage3_unit3_conv2(stage3_unit3_conv2_pad)
        stage3_unit3_bn3 = self.stage3_unit3_bn3(stage3_unit3_conv2)
        plus18          = stage3_unit3_bn3 + plus17
        stage3_unit4_bn1 = self.stage3_unit4_bn1(plus18)
        stage3_unit4_conv1_pad = F.pad(stage3_unit4_bn1, (1, 1, 1, 1))
        stage3_unit4_conv1 = self.stage3_unit4_conv1(stage3_unit4_conv1_pad)
        stage3_unit4_bn2 = self.stage3_unit4_bn2(stage3_unit4_conv1)
        stage3_unit4_relu1 = F.prelu(stage3_unit4_bn2, np2tensor(_weights_dict['stage3_unit4_relu1']['weights']))
        stage3_unit4_conv2_pad = F.pad(stage3_unit4_relu1, (1, 1, 1, 1))
        stage3_unit4_conv2 = self.stage3_unit4_conv2(stage3_unit4_conv2_pad)
        stage3_unit4_bn3 = self.stage3_unit4_bn3(stage3_unit4_conv2)
        plus19          = stage3_unit4_bn3 + plus18
        stage3_unit5_bn1 = self.stage3_unit5_bn1(plus19)
        stage3_unit5_conv1_pad = F.pad(stage3_unit5_bn1, (1, 1, 1, 1))
        stage3_unit5_conv1 = self.stage3_unit5_conv1(stage3_unit5_conv1_pad)
        stage3_unit5_bn2 = self.stage3_unit5_bn2(stage3_unit5_conv1)
        stage3_unit5_relu1 = F.prelu(stage3_unit5_bn2, np2tensor(_weights_dict['stage3_unit5_relu1']['weights']))
        stage3_unit5_conv2_pad = F.pad(stage3_unit5_relu1, (1, 1, 1, 1))
        stage3_unit5_conv2 = self.stage3_unit5_conv2(stage3_unit5_conv2_pad)
        stage3_unit5_bn3 = self.stage3_unit5_bn3(stage3_unit5_conv2)
        plus20          = stage3_unit5_bn3 + plus19
        stage3_unit6_bn1 = self.stage3_unit6_bn1(plus20)
        stage3_unit6_conv1_pad = F.pad(stage3_unit6_bn1, (1, 1, 1, 1))
        stage3_unit6_conv1 = self.stage3_unit6_conv1(stage3_unit6_conv1_pad)
        stage3_unit6_bn2 = self.stage3_unit6_bn2(stage3_unit6_conv1)
        stage3_unit6_relu1 = F.prelu(stage3_unit6_bn2, np2tensor(_weights_dict['stage3_unit6_relu1']['weights']))
        stage3_unit6_conv2_pad = F.pad(stage3_unit6_relu1, (1, 1, 1, 1))
        stage3_unit6_conv2 = self.stage3_unit6_conv2(stage3_unit6_conv2_pad)
        stage3_unit6_bn3 = self.stage3_unit6_bn3(stage3_unit6_conv2)
        plus21          = stage3_unit6_bn3 + plus20
        stage3_unit7_bn1 = self.stage3_unit7_bn1(plus21)
        stage3_unit7_conv1_pad = F.pad(stage3_unit7_bn1, (1, 1, 1, 1))
        stage3_unit7_conv1 = self.stage3_unit7_conv1(stage3_unit7_conv1_pad)
        stage3_unit7_bn2 = self.stage3_unit7_bn2(stage3_unit7_conv1)
        stage3_unit7_relu1 = F.prelu(stage3_unit7_bn2, np2tensor(_weights_dict['stage3_unit7_relu1']['weights']))
        stage3_unit7_conv2_pad = F.pad(stage3_unit7_relu1, (1, 1, 1, 1))
        stage3_unit7_conv2 = self.stage3_unit7_conv2(stage3_unit7_conv2_pad)
        stage3_unit7_bn3 = self.stage3_unit7_bn3(stage3_unit7_conv2)
        plus22          = stage3_unit7_bn3 + plus21
        stage3_unit8_bn1 = self.stage3_unit8_bn1(plus22)
        stage3_unit8_conv1_pad = F.pad(stage3_unit8_bn1, (1, 1, 1, 1))
        stage3_unit8_conv1 = self.stage3_unit8_conv1(stage3_unit8_conv1_pad)
        stage3_unit8_bn2 = self.stage3_unit8_bn2(stage3_unit8_conv1)
        stage3_unit8_relu1 = F.prelu(stage3_unit8_bn2, np2tensor(_weights_dict['stage3_unit8_relu1']['weights']))
        stage3_unit8_conv2_pad = F.pad(stage3_unit8_relu1, (1, 1, 1, 1))
        stage3_unit8_conv2 = self.stage3_unit8_conv2(stage3_unit8_conv2_pad)
        stage3_unit8_bn3 = self.stage3_unit8_bn3(stage3_unit8_conv2)
        plus23          = stage3_unit8_bn3 + plus22
        stage3_unit9_bn1 = self.stage3_unit9_bn1(plus23)
        stage3_unit9_conv1_pad = F.pad(stage3_unit9_bn1, (1, 1, 1, 1))
        stage3_unit9_conv1 = self.stage3_unit9_conv1(stage3_unit9_conv1_pad)
        stage3_unit9_bn2 = self.stage3_unit9_bn2(stage3_unit9_conv1)
        stage3_unit9_relu1 = F.prelu(stage3_unit9_bn2, np2tensor(_weights_dict['stage3_unit9_relu1']['weights']))
        stage3_unit9_conv2_pad = F.pad(stage3_unit9_relu1, (1, 1, 1, 1))
        stage3_unit9_conv2 = self.stage3_unit9_conv2(stage3_unit9_conv2_pad)
        stage3_unit9_bn3 = self.stage3_unit9_bn3(stage3_unit9_conv2)
        plus24          = stage3_unit9_bn3 + plus23
        stage3_unit10_bn1 = self.stage3_unit10_bn1(plus24)
        stage3_unit10_conv1_pad = F.pad(stage3_unit10_bn1, (1, 1, 1, 1))
        stage3_unit10_conv1 = self.stage3_unit10_conv1(stage3_unit10_conv1_pad)
        stage3_unit10_bn2 = self.stage3_unit10_bn2(stage3_unit10_conv1)
        stage3_unit10_relu1 = F.prelu(stage3_unit10_bn2, np2tensor(_weights_dict['stage3_unit10_relu1']['weights']))
        stage3_unit10_conv2_pad = F.pad(stage3_unit10_relu1, (1, 1, 1, 1))
        stage3_unit10_conv2 = self.stage3_unit10_conv2(stage3_unit10_conv2_pad)
        stage3_unit10_bn3 = self.stage3_unit10_bn3(stage3_unit10_conv2)
        plus25          = stage3_unit10_bn3 + plus24
        stage3_unit11_bn1 = self.stage3_unit11_bn1(plus25)
        stage3_unit11_conv1_pad = F.pad(stage3_unit11_bn1, (1, 1, 1, 1))
        stage3_unit11_conv1 = self.stage3_unit11_conv1(stage3_unit11_conv1_pad)
        stage3_unit11_bn2 = self.stage3_unit11_bn2(stage3_unit11_conv1)
        stage3_unit11_relu1 = F.prelu(stage3_unit11_bn2, np2tensor(_weights_dict['stage3_unit11_relu1']['weights']))
        stage3_unit11_conv2_pad = F.pad(stage3_unit11_relu1, (1, 1, 1, 1))
        stage3_unit11_conv2 = self.stage3_unit11_conv2(stage3_unit11_conv2_pad)
        stage3_unit11_bn3 = self.stage3_unit11_bn3(stage3_unit11_conv2)
        plus26          = stage3_unit11_bn3 + plus25
        stage3_unit12_bn1 = self.stage3_unit12_bn1(plus26)
        stage3_unit12_conv1_pad = F.pad(stage3_unit12_bn1, (1, 1, 1, 1))
        stage3_unit12_conv1 = self.stage3_unit12_conv1(stage3_unit12_conv1_pad)
        stage3_unit12_bn2 = self.stage3_unit12_bn2(stage3_unit12_conv1)
        stage3_unit12_relu1 = F.prelu(stage3_unit12_bn2, np2tensor(_weights_dict['stage3_unit12_relu1']['weights']))
        stage3_unit12_conv2_pad = F.pad(stage3_unit12_relu1, (1, 1, 1, 1))
        stage3_unit12_conv2 = self.stage3_unit12_conv2(stage3_unit12_conv2_pad)
        stage3_unit12_bn3 = self.stage3_unit12_bn3(stage3_unit12_conv2)
        plus27          = stage3_unit12_bn3 + plus26
        stage3_unit13_bn1 = self.stage3_unit13_bn1(plus27)
        stage3_unit13_conv1_pad = F.pad(stage3_unit13_bn1, (1, 1, 1, 1))
        stage3_unit13_conv1 = self.stage3_unit13_conv1(stage3_unit13_conv1_pad)
        stage3_unit13_bn2 = self.stage3_unit13_bn2(stage3_unit13_conv1)
        stage3_unit13_relu1 = F.prelu(stage3_unit13_bn2, np2tensor(_weights_dict['stage3_unit13_relu1']['weights']))
        stage3_unit13_conv2_pad = F.pad(stage3_unit13_relu1, (1, 1, 1, 1))
        stage3_unit13_conv2 = self.stage3_unit13_conv2(stage3_unit13_conv2_pad)
        stage3_unit13_bn3 = self.stage3_unit13_bn3(stage3_unit13_conv2)
        plus28          = stage3_unit13_bn3 + plus27
        stage3_unit14_bn1 = self.stage3_unit14_bn1(plus28)
        stage3_unit14_conv1_pad = F.pad(stage3_unit14_bn1, (1, 1, 1, 1))
        stage3_unit14_conv1 = self.stage3_unit14_conv1(stage3_unit14_conv1_pad)
        stage3_unit14_bn2 = self.stage3_unit14_bn2(stage3_unit14_conv1)
        stage3_unit14_relu1 = F.prelu(stage3_unit14_bn2, np2tensor(_weights_dict['stage3_unit14_relu1']['weights']))
        stage3_unit14_conv2_pad = F.pad(stage3_unit14_relu1, (1, 1, 1, 1))
        stage3_unit14_conv2 = self.stage3_unit14_conv2(stage3_unit14_conv2_pad)
        stage3_unit14_bn3 = self.stage3_unit14_bn3(stage3_unit14_conv2)
        plus29          = stage3_unit14_bn3 + plus28
        stage3_unit15_bn1 = self.stage3_unit15_bn1(plus29)
        stage3_unit15_conv1_pad = F.pad(stage3_unit15_bn1, (1, 1, 1, 1))
        stage3_unit15_conv1 = self.stage3_unit15_conv1(stage3_unit15_conv1_pad)
        stage3_unit15_bn2 = self.stage3_unit15_bn2(stage3_unit15_conv1)
        stage3_unit15_relu1 = F.prelu(stage3_unit15_bn2, np2tensor(_weights_dict['stage3_unit15_relu1']['weights']))
        stage3_unit15_conv2_pad = F.pad(stage3_unit15_relu1, (1, 1, 1, 1))
        stage3_unit15_conv2 = self.stage3_unit15_conv2(stage3_unit15_conv2_pad)
        stage3_unit15_bn3 = self.stage3_unit15_bn3(stage3_unit15_conv2)
        plus30          = stage3_unit15_bn3 + plus29
        stage3_unit16_bn1 = self.stage3_unit16_bn1(plus30)
        stage3_unit16_conv1_pad = F.pad(stage3_unit16_bn1, (1, 1, 1, 1))
        stage3_unit16_conv1 = self.stage3_unit16_conv1(stage3_unit16_conv1_pad)
        stage3_unit16_bn2 = self.stage3_unit16_bn2(stage3_unit16_conv1)
        stage3_unit16_relu1 = F.prelu(stage3_unit16_bn2, np2tensor(_weights_dict['stage3_unit16_relu1']['weights']))
        stage3_unit16_conv2_pad = F.pad(stage3_unit16_relu1, (1, 1, 1, 1))
        stage3_unit16_conv2 = self.stage3_unit16_conv2(stage3_unit16_conv2_pad)
        stage3_unit16_bn3 = self.stage3_unit16_bn3(stage3_unit16_conv2)
        plus31          = stage3_unit16_bn3 + plus30
        stage3_unit17_bn1 = self.stage3_unit17_bn1(plus31)
        stage3_unit17_conv1_pad = F.pad(stage3_unit17_bn1, (1, 1, 1, 1))
        stage3_unit17_conv1 = self.stage3_unit17_conv1(stage3_unit17_conv1_pad)
        stage3_unit17_bn2 = self.stage3_unit17_bn2(stage3_unit17_conv1)
        stage3_unit17_relu1 = F.prelu(stage3_unit17_bn2, np2tensor(_weights_dict['stage3_unit17_relu1']['weights']))
        stage3_unit17_conv2_pad = F.pad(stage3_unit17_relu1, (1, 1, 1, 1))
        stage3_unit17_conv2 = self.stage3_unit17_conv2(stage3_unit17_conv2_pad)
        stage3_unit17_bn3 = self.stage3_unit17_bn3(stage3_unit17_conv2)
        plus32          = stage3_unit17_bn3 + plus31
        stage3_unit18_bn1 = self.stage3_unit18_bn1(plus32)
        stage3_unit18_conv1_pad = F.pad(stage3_unit18_bn1, (1, 1, 1, 1))
        stage3_unit18_conv1 = self.stage3_unit18_conv1(stage3_unit18_conv1_pad)
        stage3_unit18_bn2 = self.stage3_unit18_bn2(stage3_unit18_conv1)
        stage3_unit18_relu1 = F.prelu(stage3_unit18_bn2, np2tensor(_weights_dict['stage3_unit18_relu1']['weights']))
        stage3_unit18_conv2_pad = F.pad(stage3_unit18_relu1, (1, 1, 1, 1))
        stage3_unit18_conv2 = self.stage3_unit18_conv2(stage3_unit18_conv2_pad)
        stage3_unit18_bn3 = self.stage3_unit18_bn3(stage3_unit18_conv2)
        plus33          = stage3_unit18_bn3 + plus32
        stage3_unit19_bn1 = self.stage3_unit19_bn1(plus33)
        stage3_unit19_conv1_pad = F.pad(stage3_unit19_bn1, (1, 1, 1, 1))
        stage3_unit19_conv1 = self.stage3_unit19_conv1(stage3_unit19_conv1_pad)
        stage3_unit19_bn2 = self.stage3_unit19_bn2(stage3_unit19_conv1)
        stage3_unit19_relu1 = F.prelu(stage3_unit19_bn2, np2tensor(_weights_dict['stage3_unit19_relu1']['weights']))
        stage3_unit19_conv2_pad = F.pad(stage3_unit19_relu1, (1, 1, 1, 1))
        stage3_unit19_conv2 = self.stage3_unit19_conv2(stage3_unit19_conv2_pad)
        stage3_unit19_bn3 = self.stage3_unit19_bn3(stage3_unit19_conv2)
        plus34          = stage3_unit19_bn3 + plus33
        stage3_unit20_bn1 = self.stage3_unit20_bn1(plus34)
        stage3_unit20_conv1_pad = F.pad(stage3_unit20_bn1, (1, 1, 1, 1))
        stage3_unit20_conv1 = self.stage3_unit20_conv1(stage3_unit20_conv1_pad)
        stage3_unit20_bn2 = self.stage3_unit20_bn2(stage3_unit20_conv1)
        stage3_unit20_relu1 = F.prelu(stage3_unit20_bn2, np2tensor(_weights_dict['stage3_unit20_relu1']['weights']))
        stage3_unit20_conv2_pad = F.pad(stage3_unit20_relu1, (1, 1, 1, 1))
        stage3_unit20_conv2 = self.stage3_unit20_conv2(stage3_unit20_conv2_pad)
        stage3_unit20_bn3 = self.stage3_unit20_bn3(stage3_unit20_conv2)
        plus35          = stage3_unit20_bn3 + plus34
        stage3_unit21_bn1 = self.stage3_unit21_bn1(plus35)
        stage3_unit21_conv1_pad = F.pad(stage3_unit21_bn1, (1, 1, 1, 1))
        stage3_unit21_conv1 = self.stage3_unit21_conv1(stage3_unit21_conv1_pad)
        stage3_unit21_bn2 = self.stage3_unit21_bn2(stage3_unit21_conv1)
        stage3_unit21_relu1 = F.prelu(stage3_unit21_bn2, np2tensor(_weights_dict['stage3_unit21_relu1']['weights']))
        stage3_unit21_conv2_pad = F.pad(stage3_unit21_relu1, (1, 1, 1, 1))
        stage3_unit21_conv2 = self.stage3_unit21_conv2(stage3_unit21_conv2_pad)
        stage3_unit21_bn3 = self.stage3_unit21_bn3(stage3_unit21_conv2)
        plus36          = stage3_unit21_bn3 + plus35
        stage3_unit22_bn1 = self.stage3_unit22_bn1(plus36)
        stage3_unit22_conv1_pad = F.pad(stage3_unit22_bn1, (1, 1, 1, 1))
        stage3_unit22_conv1 = self.stage3_unit22_conv1(stage3_unit22_conv1_pad)
        stage3_unit22_bn2 = self.stage3_unit22_bn2(stage3_unit22_conv1)
        stage3_unit22_relu1 = F.prelu(stage3_unit22_bn2, np2tensor(_weights_dict['stage3_unit22_relu1']['weights']))
        stage3_unit22_conv2_pad = F.pad(stage3_unit22_relu1, (1, 1, 1, 1))
        stage3_unit22_conv2 = self.stage3_unit22_conv2(stage3_unit22_conv2_pad)
        stage3_unit22_bn3 = self.stage3_unit22_bn3(stage3_unit22_conv2)
        plus37          = stage3_unit22_bn3 + plus36
        stage3_unit23_bn1 = self.stage3_unit23_bn1(plus37)
        stage3_unit23_conv1_pad = F.pad(stage3_unit23_bn1, (1, 1, 1, 1))
        stage3_unit23_conv1 = self.stage3_unit23_conv1(stage3_unit23_conv1_pad)
        stage3_unit23_bn2 = self.stage3_unit23_bn2(stage3_unit23_conv1)
        stage3_unit23_relu1 = F.prelu(stage3_unit23_bn2, np2tensor(_weights_dict['stage3_unit23_relu1']['weights']))
        stage3_unit23_conv2_pad = F.pad(stage3_unit23_relu1, (1, 1, 1, 1))
        stage3_unit23_conv2 = self.stage3_unit23_conv2(stage3_unit23_conv2_pad)
        stage3_unit23_bn3 = self.stage3_unit23_bn3(stage3_unit23_conv2)
        plus38          = stage3_unit23_bn3 + plus37
        stage3_unit24_bn1 = self.stage3_unit24_bn1(plus38)
        stage3_unit24_conv1_pad = F.pad(stage3_unit24_bn1, (1, 1, 1, 1))
        stage3_unit24_conv1 = self.stage3_unit24_conv1(stage3_unit24_conv1_pad)
        stage3_unit24_bn2 = self.stage3_unit24_bn2(stage3_unit24_conv1)
        stage3_unit24_relu1 = F.prelu(stage3_unit24_bn2, np2tensor(_weights_dict['stage3_unit24_relu1']['weights']))
        stage3_unit24_conv2_pad = F.pad(stage3_unit24_relu1, (1, 1, 1, 1))
        stage3_unit24_conv2 = self.stage3_unit24_conv2(stage3_unit24_conv2_pad)
        stage3_unit24_bn3 = self.stage3_unit24_bn3(stage3_unit24_conv2)
        plus39          = stage3_unit24_bn3 + plus38
        stage3_unit25_bn1 = self.stage3_unit25_bn1(plus39)
        stage3_unit25_conv1_pad = F.pad(stage3_unit25_bn1, (1, 1, 1, 1))
        stage3_unit25_conv1 = self.stage3_unit25_conv1(stage3_unit25_conv1_pad)
        stage3_unit25_bn2 = self.stage3_unit25_bn2(stage3_unit25_conv1)
        stage3_unit25_relu1 = F.prelu(stage3_unit25_bn2, np2tensor(_weights_dict['stage3_unit25_relu1']['weights']))
        stage3_unit25_conv2_pad = F.pad(stage3_unit25_relu1, (1, 1, 1, 1))
        stage3_unit25_conv2 = self.stage3_unit25_conv2(stage3_unit25_conv2_pad)
        stage3_unit25_bn3 = self.stage3_unit25_bn3(stage3_unit25_conv2)
        plus40          = stage3_unit25_bn3 + plus39
        stage3_unit26_bn1 = self.stage3_unit26_bn1(plus40)
        stage3_unit26_conv1_pad = F.pad(stage3_unit26_bn1, (1, 1, 1, 1))
        stage3_unit26_conv1 = self.stage3_unit26_conv1(stage3_unit26_conv1_pad)
        stage3_unit26_bn2 = self.stage3_unit26_bn2(stage3_unit26_conv1)
        stage3_unit26_relu1 = F.prelu(stage3_unit26_bn2, np2tensor(_weights_dict['stage3_unit26_relu1']['weights']))
        stage3_unit26_conv2_pad = F.pad(stage3_unit26_relu1, (1, 1, 1, 1))
        stage3_unit26_conv2 = self.stage3_unit26_conv2(stage3_unit26_conv2_pad)
        stage3_unit26_bn3 = self.stage3_unit26_bn3(stage3_unit26_conv2)
        plus41          = stage3_unit26_bn3 + plus40
        stage3_unit27_bn1 = self.stage3_unit27_bn1(plus41)
        stage3_unit27_conv1_pad = F.pad(stage3_unit27_bn1, (1, 1, 1, 1))
        stage3_unit27_conv1 = self.stage3_unit27_conv1(stage3_unit27_conv1_pad)
        stage3_unit27_bn2 = self.stage3_unit27_bn2(stage3_unit27_conv1)
        stage3_unit27_relu1 = F.prelu(stage3_unit27_bn2, np2tensor(_weights_dict['stage3_unit27_relu1']['weights']))
        stage3_unit27_conv2_pad = F.pad(stage3_unit27_relu1, (1, 1, 1, 1))
        stage3_unit27_conv2 = self.stage3_unit27_conv2(stage3_unit27_conv2_pad)
        stage3_unit27_bn3 = self.stage3_unit27_bn3(stage3_unit27_conv2)
        plus42          = stage3_unit27_bn3 + plus41
        stage3_unit28_bn1 = self.stage3_unit28_bn1(plus42)
        stage3_unit28_conv1_pad = F.pad(stage3_unit28_bn1, (1, 1, 1, 1))
        stage3_unit28_conv1 = self.stage3_unit28_conv1(stage3_unit28_conv1_pad)
        stage3_unit28_bn2 = self.stage3_unit28_bn2(stage3_unit28_conv1)
        stage3_unit28_relu1 = F.prelu(stage3_unit28_bn2, np2tensor(_weights_dict['stage3_unit28_relu1']['weights']))
        stage3_unit28_conv2_pad = F.pad(stage3_unit28_relu1, (1, 1, 1, 1))
        stage3_unit28_conv2 = self.stage3_unit28_conv2(stage3_unit28_conv2_pad)
        stage3_unit28_bn3 = self.stage3_unit28_bn3(stage3_unit28_conv2)
        plus43          = stage3_unit28_bn3 + plus42
        stage3_unit29_bn1 = self.stage3_unit29_bn1(plus43)
        stage3_unit29_conv1_pad = F.pad(stage3_unit29_bn1, (1, 1, 1, 1))
        stage3_unit29_conv1 = self.stage3_unit29_conv1(stage3_unit29_conv1_pad)
        stage3_unit29_bn2 = self.stage3_unit29_bn2(stage3_unit29_conv1)
        stage3_unit29_relu1 = F.prelu(stage3_unit29_bn2, np2tensor(_weights_dict['stage3_unit29_relu1']['weights']))
        stage3_unit29_conv2_pad = F.pad(stage3_unit29_relu1, (1, 1, 1, 1))
        stage3_unit29_conv2 = self.stage3_unit29_conv2(stage3_unit29_conv2_pad)
        stage3_unit29_bn3 = self.stage3_unit29_bn3(stage3_unit29_conv2)
        plus44          = stage3_unit29_bn3 + plus43
        stage3_unit30_bn1 = self.stage3_unit30_bn1(plus44)
        stage3_unit30_conv1_pad = F.pad(stage3_unit30_bn1, (1, 1, 1, 1))
        stage3_unit30_conv1 = self.stage3_unit30_conv1(stage3_unit30_conv1_pad)
        stage3_unit30_bn2 = self.stage3_unit30_bn2(stage3_unit30_conv1)
        stage3_unit30_relu1 = F.prelu(stage3_unit30_bn2, np2tensor(_weights_dict['stage3_unit30_relu1']['weights']))
        stage3_unit30_conv2_pad = F.pad(stage3_unit30_relu1, (1, 1, 1, 1))
        stage3_unit30_conv2 = self.stage3_unit30_conv2(stage3_unit30_conv2_pad)
        stage3_unit30_bn3 = self.stage3_unit30_bn3(stage3_unit30_conv2)
        plus45          = stage3_unit30_bn3 + plus44
        stage4_unit1_bn1 = self.stage4_unit1_bn1(plus45)
        stage4_unit1_conv1sc = self.stage4_unit1_conv1sc(plus45)
        stage4_unit1_conv1_pad = F.pad(stage4_unit1_bn1, (1, 1, 1, 1))
        stage4_unit1_conv1 = self.stage4_unit1_conv1(stage4_unit1_conv1_pad)
        stage4_unit1_sc = self.stage4_unit1_sc(stage4_unit1_conv1sc)
        stage4_unit1_bn2 = self.stage4_unit1_bn2(stage4_unit1_conv1)
        stage4_unit1_relu1 = F.prelu(stage4_unit1_bn2, np2tensor(_weights_dict['stage4_unit1_relu1']['weights']))
        stage4_unit1_conv2_pad = F.pad(stage4_unit1_relu1, (1, 1, 1, 1))
        stage4_unit1_conv2 = self.stage4_unit1_conv2(stage4_unit1_conv2_pad)
        stage4_unit1_bn3 = self.stage4_unit1_bn3(stage4_unit1_conv2)
        stage4_unit1_bn3 = self.CI_46(stage4_unit1_bn3)
        plus46          = stage4_unit1_bn3 + stage4_unit1_sc
        # plus46 = self.CI_46(plus46)
        stage4_unit2_bn1 = self.stage4_unit2_bn1(plus46)
        stage4_unit2_conv1_pad = F.pad(stage4_unit2_bn1, (1, 1, 1, 1))
        stage4_unit2_conv1 = self.stage4_unit2_conv1(stage4_unit2_conv1_pad)
        stage4_unit2_bn2 = self.stage4_unit2_bn2(stage4_unit2_conv1)
        stage4_unit2_relu1 = F.prelu(stage4_unit2_bn2, np2tensor(_weights_dict['stage4_unit2_relu1']['weights']))
        stage4_unit2_conv2_pad = F.pad(stage4_unit2_relu1, (1, 1, 1, 1))
        stage4_unit2_conv2 = self.stage4_unit2_conv2(stage4_unit2_conv2_pad)
        stage4_unit2_bn3 = self.stage4_unit2_bn3(stage4_unit2_conv2)
        stage4_unit2_bn3 = self.CI_47(stage4_unit2_bn3)
        plus47          = stage4_unit2_bn3 + plus46
        # plus47 = self.CI_47(plus47)
        stage4_unit3_bn1 = self.stage4_unit3_bn1(plus47)
        stage4_unit3_conv1_pad = F.pad(stage4_unit3_bn1, (1, 1, 1, 1))
        stage4_unit3_conv1 = self.stage4_unit3_conv1(stage4_unit3_conv1_pad)
        stage4_unit3_bn2 = self.stage4_unit3_bn2(stage4_unit3_conv1)
        stage4_unit3_relu1 = F.prelu(stage4_unit3_bn2, np2tensor(_weights_dict['stage4_unit3_relu1']['weights']))
        stage4_unit3_conv2_pad = F.pad(stage4_unit3_relu1, (1, 1, 1, 1))
        stage4_unit3_conv2 = self.stage4_unit3_conv2(stage4_unit3_conv2_pad)
        stage4_unit3_bn3 = self.stage4_unit3_bn3(stage4_unit3_conv2)
        stage4_unit3_bn3 = self.CI_48(stage4_unit3_bn3)
        plus48          = stage4_unit3_bn3 + plus47
        # plus48 = self.CI_48(plus48)
        bn1             = self.bn1(plus48)
        dropout0        = F.dropout(input = bn1, p = 0.4000000059604645, training = self.training, inplace = True)
        pre_fc1         = self.pre_fc1(dropout0.view(dropout0.size(0), -1))
        fc1             = self.fc1(pre_fc1)
        # return fc1
        return fc1, plus48
        # return fc1, plus20, plus30, plus45


    @staticmethod
    def __batch_normalization(dim, name, **kwargs):
        if   dim == 0 or dim == 1:  layer = nn.BatchNorm1d(**kwargs)
        elif dim == 2:  layer = nn.BatchNorm2d(**kwargs)
        elif dim == 3:  layer = nn.BatchNorm3d(**kwargs)
        else:           raise NotImplementedError()

        if 'scale' in _weights_dict[name]:
            layer.state_dict()['weight'].copy_(torch.from_numpy(_weights_dict[name]['scale']))
        else:
            layer.weight.data.fill_(1)

        if 'bias' in _weights_dict[name]:
            layer.state_dict()['bias'].copy_(torch.from_numpy(_weights_dict[name]['bias']))
        else:
            layer.bias.data.fill_(0)

        layer.state_dict()['running_mean'].copy_(torch.from_numpy(_weights_dict[name]['mean']))
        layer.state_dict()['running_var'].copy_(torch.from_numpy(_weights_dict[name]['var']))
        return layer

    @staticmethod
    def __dense(name, **kwargs):
        layer = nn.Linear(**kwargs)
        layer.state_dict()['weight'].copy_(torch.from_numpy(_weights_dict[name]['weights']))
        if 'bias' in _weights_dict[name]:
            layer.state_dict()['bias'].copy_(torch.from_numpy(_weights_dict[name]['bias']))
        return layer

    @staticmethod
    def __conv(dim, name, **kwargs):
        if   dim == 1:  layer = nn.Conv1d(**kwargs)
        elif dim == 2:  layer = nn.Conv2d(**kwargs)
        elif dim == 3:  layer = nn.Conv3d(**kwargs)
        else:           raise NotImplementedError()

        layer.state_dict()['weight'].copy_(torch.from_numpy(_weights_dict[name]['weights']))
        if 'bias' in _weights_dict[name]:
            layer.state_dict()['bias'].copy_(torch.from_numpy(_weights_dict[name]['bias']))
        return layer


torch_resnet101.py
garynlfd/KFC
from sklearn.metrics import roc_curve,auc
from dataset import *
from models_multi_task import *
from torch.optim import SGD
from losses import *
import argparse
from tqdm import tqdm
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import numpy as np

import warnings; warnings.filterwarnings("ignore")

race_dict={'AA':np.array([0], dtype=np.long),'A':np.array([1], dtype=np.long),'C':np.array([2], dtype=np.long),'I':np.array([3], dtype=np.long),\
        'AA&AA':np.array([4], dtype=np.long),'AA&A':np.array([5], dtype=np.long),'AA&C':np.array([6], dtype=np.long),'AA&I':np.array([7], dtype=np.long),\
        'A&AA':np.array([8], dtype=np.long),'A&A':np.array([9], dtype=np.long),'A&C':np.array([10], dtype=np.long),'A&I':np.array([11], dtype=np.long),\
        'C&AA':np.array([12], dtype=np.long),'C&A':np.array([13], dtype=np.long),'C&C':np.array([14], dtype=np.long),'C&I':np.array([15], dtype=np.long),\
        'I&AA':np.array([16], dtype=np.long),'I&A':np.array([17], dtype=np.long),'I&C':np.array([18], dtype=np.long),'I&I':np.array([19], dtype=np.long)}


def training(args):

    batch_size=args.batch_size
    val_batch_size=args.batch_size
    epochs=args.epochs
    steps_per_epoch=600
    save_path=os.path.join(args.save_path,"weights_Cornell_multi")
    beta=args.beta
    log_path=args.log_path


    train_dataset=FIW(os.path.join(args.sample,"./aligned_dataset/mixed_dataset/mixed_dataset_train.txt"))
    val_dataset=FIW(os.path.join(args.sample,"./aligned_dataset/mixed_dataset/mixed_dataset_val_choose_cross.txt"))

    train_loader=DataLoader(train_dataset,batch_size=batch_size,num_workers=1,pin_memory=False)
    val_loader = DataLoader(val_dataset, batch_size=val_batch_size, num_workers=1, pin_memory=False)

    model=MultiTask().cuda()

    optimizer_model = SGD(model.parameters(), lr=1e-4, momentum=0.9)
    max_auc=0.0
    min_std=100

    criterion_CE=torch.nn.CrossEntropyLoss()

    for epoch_i in range(epochs):
        mylog("\n*************",path=log_path)
        mylog('epoch ' + str(epoch_i + 1),path=log_path)
        total_loss_epoch = 0
        kinship_loss_epoch = 0
        race_loss_epoch = 0
        face_recognition_loss_epoch = 0
        margin_list_sum = [0, 0, 0, 0]
        margin_list_appearance= [0, 0, 0, 0]
        model.train()
        for index_i, data in enumerate(tqdm(train_loader)): # this loop loops for 1 batch
            image1,image2,kinship,labels,races=data # one batch includes 25 image pairs
            r1,r2,e1,e2,x1,x2,bias_map,bias_pair= model([image1,image2])
            kinship_loss, margin_list = contrastive_loss(x1,x2,kinship,races,bias_map,bias_pair,beta=beta)
            race_loss = criterion_CE(r1, races) + criterion_CE(r2, races)

            for i in range(4):
                if margin_list[i] != 0: 
                    margin_list_appearance[i] += 1
            for i in range(4):
                margin_list_sum[i] += margin_list[i]

            loss = kinship_loss + race_loss


            optimizer_model.zero_grad()
            loss.backward()
            optimizer_model.step()

            kinship_loss_epoch += kinship_loss.item()
            race_loss_epoch += race_loss.item()
            total_loss_epoch += loss.item()
            
            if (index_i+1)==steps_per_epoch:
                break

        use_sample=(epoch_i+1)*batch_size*steps_per_epoch
        print(use_sample)
        train_dataset.set_bias(use_sample)

        for i in range(4):
            if margin_list_appearance[i] != 0:
                margin_list_sum[i] /= margin_list_appearance[i]

        mylog("total_loss:" + "%.6f" % (total_loss_epoch / steps_per_epoch),path=log_path)
        mylog("kinship_loss:" + "%.6f" % (kinship_loss_epoch / steps_per_epoch),path=log_path)
        mylog("race_loss:" + "%.6f" % (race_loss_epoch / steps_per_epoch),path=log_path)
        mylog("race margin: ", margin_list_sum, path=log_path)
        model.eval()
        with torch.no_grad():
            auc,std = val_model(model, val_loader)
        mylog("auc is %.6f "% auc,path=log_path)
        mylog("std is %.6f "% std,path=log_path)
        if max_auc < auc:
            mylog("auc improve from :" + "%.6f" % max_auc + " to %.6f" % auc,path=log_path)
            max_auc=auc
            mylog("save model " + save_path,path=log_path)
            model.save(save_path,"_auc_best")
        else:
            mylog("auc did not improve from %.6f" % float(max_auc),path=log_path)
            if((epoch_i+1)%10==0):
                model.save(save_path,epoch_i+1)
        if min_std > std:
            mylog("std improve from :" + "%.6f" % min_std + " to %.6f" % std,path=log_path)
            min_std=std
            mylog("save model " + save_path,path=log_path)
            model.save(save_path,"_std_best")

def save_model(model,path):
    torch.save(model.state_dict(),path)

def val_model(model, val_loader):
    y_true = []
    y_pred = []
    race_label=[]
    race_table={'A':0,'AA':0,'C':0,"I":0}
    race_total={'A':0,'AA':0,'C':0,"I":0}
    for img1, img2, kinship, labels, races in val_loader:
        r1,r2,e1,e2,x1,x2,bias_map,bias_pair=model([img1.cuda(),img2.cuda()])
        y_pred.extend(torch.cosine_similarity(e1,e2,dim=1).cpu().detach().numpy().tolist())
        y_true.extend(labels.cpu().detach().numpy().tolist())
        race_label.extend(races.cpu().detach().numpy().tolist())
    fpr, tpr, threshold_ = roc_curve(y_true, y_pred)
    maxindex = (tpr - fpr).tolist().index(max(tpr - fpr))
    threshold = threshold_[maxindex]

    pred_cos= torch.tensor(y_pred)>threshold
    for i,pred in enumerate(pred_cos):
        if(pred==y_true[i]):
            race_table[list(race_dict.keys())[list(race_dict.values()).index(int(race_label[i]))].split('&')[0]]+=1
        race_total[list(race_dict.keys())[list(race_dict.values()).index(int(race_label[i]))].split('&')[0]]+=1
        
    acc=np.array([race_table[i]/race_total[i] for i in ['A','AA','C','I']])
    std=np.std(acc)

    return auc(fpr,tpr),std


if __name__=="__main__":
    parser = argparse.ArgumentParser(description="train")
    parser.add_argument("--batch_size", type=int, default=25,help="batch size default 25")
    parser.add_argument( "--sample", type=str, help="sample root")
    parser.add_argument( "--save_path",  type=str, help="model save path")
    parser.add_argument( "--epochs", type=int,default=100, help="epochs number default 100")
    parser.add_argument( "--beta", default=0.08, type=float, help="beta default 0.08")
    parser.add_argument( "--log_path", default="./log.txt", type=str, help="log path default log.txt")
    parser.add_argument( "--gpu", default="1", type=str, help="gpu id you use")
    args = parser.parse_args()
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
    torch.multiprocessing.set_start_method("spawn")
    set_seed(seed=100)
    training(args)

train.py
garynlfd/KFC