<kaggle_start><data_title>covid19-512<data_name>covid19512
<code># # Warrning
# 1. This notebook only include public testset for faster testif you submit it you will get 0 score
# 2. Change wandb API KEY in training notebook to yours
# 3. colab free GPU is about 3x faster than kaggle
# # Approach and Refferences
# > efficientnetb3a with aux loss for study + efficientnetb5 for 2class + yolov5m for image
# ## Refferences
# 1. henhttps://www.kaggle.com/c/siim-covid19-detection/discussion/246586gk's aux loss https://www.kaggle.com/c/siim-covid19-detection/discussion/240233
# 2. alien's 2 class tricks https://www.kaggle.com/c/siim-covid19-detection/discussion/246586
# 3. darian's duplicate analysis https://www.kaggle.com/c/siim-covid19-detection/discussion/240878
# ## Training notebook
# 1. drop duplicate and create mask https://www.kaggle.com/drzhuzhe/siiim-covid-stratified-k-fold-and-create-mask
# 2. training study level https://www.kaggle.com/drzhuzhe/covid19-classify efficientnetb3a 1e-3 10 ep + 1e-4 5ep CV map*0.66 score (3.76 + 3.92 + 3.85 + 3.7 + 3.6)/5 avg 3.766
# 3. training yolo https://www.kaggle.com/drzhuzhe/covid19-det?scriptVersionId=67605495 efficientnetb5 only 10 epoch  CV map score (0.4947 + 0.5103 + 0.4848 +0.4692 +0.5198)/5 avg 0.49575
# 4. 2 class https://www.kaggle.com/drzhuzhe/siim-covid19-efnb7-train-fold0-5-2class  yolov5m 15 epoch (0.869 + 0.860 + 0.882 + 0.878 + 0.876)/5 avg 0.872
# ## Experiments
# 1. study level with 6 class is doable single fold LB 0.586 +
# 2. efficientnetV2-m got low result https://www.kaggle.com/c/siim-covid19-detection/discussion/248442 according to this disscus efficientnetV2 may need larger batchsize
# 3. aux CNN head attach to block 4 get mediocre result
#
import numpy as np, pandas as pd
from glob import glob
import shutil, os
import matplotlib.pyplot as plt
from sklearn.model_selection import GroupKFold
from tqdm.notebook import tqdm
import seaborn as sns
import cv2
import sys
import math
from timeit import default_timer as timer
from datetime import datetime
from numba import cuda

sys.path.append("../input/timm-pytorch-image-models/pytorch-image-models-master")
_det_model_path = "/kaggle/input/collect-submit-model/det/"
_classify_model1_path = "/kaggle/input/covidmodels/Archive/classify-ep12/"
_classify_model2_path = "/kaggle/input/covidmodels/"
_test_files_path = "/kaggle/input/covid19512/test/"
_data_dir = "/kaggle/input/covid19512/"
meta_df = pd.read_csv(_data_dir + "meta.csv")
IMG_SIZE = 512

# # Clssification
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.parallel.data_parallel import data_parallel
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from torch.utils.data.sampler import *
import collections
from collections import defaultdict
import timm
from timm.models.efficientnet import *
import torch.cuda.amp as amp

data_dir = _data_dir
image_size = IMG_SIZE
study_name_to_predict_string = {
    "Negative for Pneumonia": "negative",
    "Typical Appearance": "typical",
    "Indeterminate Appearance": "indeterminate",
    "Atypical Appearance": "atypical",
}
study_name_to_label = {
    "Negative for Pneumonia": 0,
    "Typical Appearance": 1,
    "Indeterminate Appearance": 2,
    "Atypical Appearance": 3,
}
study_label_to_name = {v: k for k, v in study_name_to_label.items()}
num_study_label = len(study_name_to_label)


def make_fold(mode="train-0"):
    if "test" in mode:
        df_meta = pd.read_csv(data_dir + "meta.csv")
        df_valid = df_meta[df_meta["split"] == "test"].copy()
        for l in study_name_to_label.keys():
            df_valid.loc[:, l] = 0
        df_valid = df_valid.reset_index(drop=True)
        return df_valid


class SiimDataset(Dataset):
    def __init__(self, df, augment=None):
        super().__init__()
        self.df = df
        self.augment = augment
        self.length = len(df)

    def __str__(self):
        string = ""
        string += "\tlen = %d\n" % len(self)
        string += "\tdf  = %s\n" % str(self.df.shape)
        string += "\tlabel distribution\n"
        for i in range(num_study_label):
            n = self.df[study_label_to_name[i]].sum()
            string += "\t\t %d %26s: %5d (%0.4f)\n" % (
                i,
                study_label_to_name[i],
                n,
                n / len(self.df),
            )
        return string

    def __len__(self):
        return self.length

    def __getitem__(self, index):
        d = self.df.iloc[index]
        image_file = data_dir + "/test/%s.png" % (d.image_id)
        image = cv2.imread(image_file, cv2.IMREAD_GRAYSCALE)
        onehot = d[study_name_to_label.keys()].values
        mask = np.zeros_like(image)
        r = {
            "index": index,
            "d": d,
            "image": image,
            "mask": mask,
            "onehot": onehot,
        }
        if self.augment is not None:
            r = self.augment(r)
        return r


def null_collate(batch):
    collate = defaultdict(list)
    for r in batch:
        for k, v in r.items():
            collate[k].append(v)
    # ---
    batch_size = len(batch)
    onehot = np.ascontiguousarray(np.stack(collate["onehot"])).astype(np.float32)
    collate["onehot"] = torch.from_numpy(onehot)
    image = np.stack(collate["image"])
    image = image.reshape(batch_size, 1, image_size, image_size).repeat(3, 1)
    image = np.ascontiguousarray(image)
    image = image.astype(np.float32) / 255
    collate["image"] = torch.from_numpy(image)
    mask = np.stack(collate["mask"])
    mask = mask.reshape(batch_size, 1, image_size, image_size)
    mask = np.ascontiguousarray(mask)
    mask = mask.astype(np.float32) / 255
    collate["mask"] = torch.from_numpy(mask)
    return collate


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        e = efficientnet_b3a(pretrained=False, drop_rate=0.3, drop_path_rate=0.2)
        self.b0 = nn.Sequential(
            e.conv_stem,
            e.bn1,
            e.act1,
        )
        self.b1 = e.blocks[0]
        self.b2 = e.blocks[1]
        self.b3 = e.blocks[2]
        self.b4 = e.blocks[3]
        self.b5 = e.blocks[4]
        self.b6 = e.blocks[5]
        self.b7 = e.blocks[6]
        self.b8 = nn.Sequential(
            e.conv_head,  # 384, 1536
            e.bn2,
            e.act2,
        )
        self.logit = nn.Linear(1536, num_study_label)
        self.mask = nn.Sequential(
            nn.Conv2d(136, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 1, kernel_size=1, padding=0),
        )

    # @torch.cuda.amp.autocast()
    def forward(self, image):
        batch_size = len(image)
        x = 2 * image - 1  # ; print('input ',   x.shape)
        x = self.b0(x)  # ; print (x.shape)  # torch.Size([2, 40, 256, 256])
        x = self.b1(x)  # ; print (x.shape)  # torch.Size([2, 24, 256, 256])
        x = self.b2(x)  # ; print (x.shape)  # torch.Size([2, 32, 128, 128])
        x = self.b3(x)  # ; print (x.shape)  # torch.Size([2, 48, 64, 64])
        x = self.b4(x)  # ; print (x.shape)  # torch.Size([2, 96, 32, 32])
        x = self.b5(x)  # ; print (x.shape)  # torch.Size([2, 136, 32, 32])
        # ------------
        mask = self.mask(x)
        # -------------
        x = self.b6(x)  # ; print (x.shape)  # torch.Size([2, 232, 16, 16])
        x = self.b7(x)  # ; print (x.shape)  # torch.Size([2, 384, 16, 16])
        x = self.b8(x)  # ; print (x.shape)  # torch.Size([2, 1536, 16, 16])
        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)
        # x = F.dropout(x, 0.5, training=self.training)
        logit = self.logit(x)
        return logit, mask


class NetV2(nn.Module):
    def __init__(self):
        super(NetV2, self).__init__()
        # e = efficientnet_b3a(pretrained=True, drop_rate=0.3, drop_path_rate=0.2)
        # e = efficientnetv2_rw_m(pretrained=True, drop_rate=0.5, drop_path_rate=0.2)
        # e = tf_efficientnetv2_m_in21ft1k(pretrained=True, drop_rate=0.3, drop_path_rate=0.2)
        e = tf_efficientnetv2_m(pretrained=False, drop_path_rate=0.4)
        self.b0 = nn.Sequential(
            e.conv_stem,
            e.bn1,
            e.act1,
        )
        self.b1 = e.blocks[0]
        self.b2 = e.blocks[1]
        self.b3 = e.blocks[2]
        self.b4 = e.blocks[3]
        self.b5 = e.blocks[4]
        self.b6 = e.blocks[5]
        self.b7 = e.blocks[6]
        self.b8 = nn.Sequential(
            e.conv_head,  # 384, 1536
            e.bn2,
            e.act2,
        )
        self.logit = nn.Linear(1280, num_study_label)
        self.mask = nn.Sequential(
            nn.Conv2d(176, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            # nn.Conv2d(128, 128, kernel_size=3, padding=1),
            # nn.BatchNorm2d(128),
            # nn.ReLU(inplace=True),
            nn.Conv2d(128, 1, kernel_size=1, padding=0),
        )

    # @torch.cuda.amp.autocast()
    def forward(self, image):
        batch_size = len(image)
        x = 2 * image - 1  # ; print('input ',   x.shape)
        """
        rw_m
        torch.Size([2, 32, 256, 256])
        torch.Size([2, 32, 256, 256])
        torch.Size([2, 56, 128, 128])
        torch.Size([2, 80, 64, 64])
        torch.Size([2, 152, 32, 32])
        torch.Size([2, 192, 32, 32])
        torch.Size([2, 328, 16, 16])
        torch.Size([2, 2152, 16, 16])
        """
        x = self.b0(x)
        # print (x.shape)  # torch.Size([2, 32, 256, 256])
        x = self.b1(x)
        # print (x.shape)  # torch.Size([2, 32, 256, 256])
        x = self.b2(x)
        # print (x.shape)  # torch.Size([2, 56, 128, 128])
        x = self.b3(x)
        # print (x.shape)  # torch.Size([2, 80, 64, 64])
        x = self.b4(x)
        # print (x.shape)  # torch.Size([2, 152, 32, 32])
        x = self.b5(x)
        # print (x.shape)  # torch.Size([2, 192, 32, 32])
        # ------------
        mask = self.mask(x)
        # -------------
        x = self.b6(x)
        # print (x.shape)  # torch.Size([2, 328, 16, 16])
        x = self.b7(x)
        # print (x.shape)  # torch.Size([2, 512, 16, 16])
        x = self.b8(x)
        # print (x.shape)  # torch.Size([2, 2152, 16, 16])
        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)
        x = F.dropout(x, 0.5, training=self.training)
        logit = self.logit(x)
        return logit, mask


def probability_to_df_study(df_valid, probability):
    df_study = pd.DataFrame()
    df_image = df_valid.copy()
    df_study.loc[:, "id"] = df_valid.study + "_study"
    for i in range(num_study_label):
        df_study.loc[
            :, study_name_to_predict_string[study_label_to_name[i]]
        ] = probability[:, i]
        df_image.loc[
            :, study_name_to_predict_string[study_label_to_name[i]]
        ] = probability[:, i]
    df_study = df_study.groupby("id", as_index=False).mean()
    df_study.loc[:, "PredictionString"] = (
        "negative "
        + df_study.negative.apply(lambda x: "%0.6f" % x)
        + " 0 0 1 1"
        + " typical "
        + df_study.typical.apply(lambda x: "%0.6f" % x)
        + " 0 0 1 1"
        + " indeterminate "
        + df_study.indeterminate.apply(lambda x: "%0.6f" % x)
        + " 0 0 1 1"
        + " atypical "
        + df_study.atypical.apply(lambda x: "%0.6f" % x)
        + " 0 0 1 1"
    )
    df_study = df_study[["id", "PredictionString"]]
    return df_study, df_image


def do_predict(net, valid_loader, tta=["flip", "scale"]):  # flip
    valid_probability = []
    valid_num = 0
    start_timer = timer()
    for t, batch in enumerate(valid_loader):
        batch_size = len(batch["index"])
        image = batch["image"].cuda()
        onehot = batch["onehot"]
        label = onehot.argmax(-1)
        # <todo> TTA
        net.eval()
        with torch.no_grad():
            probability = []
            logit, mask = net(image)
            probability.append(F.softmax(logit, -1))
            if "flip" in tta:
                logit, mask = net(torch.flip(image, dims=(3,)))
                probability.append(F.softmax(logit, -1))
            if "scale" in tta:
                # size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None):
                logit, mask = net(
                    F.interpolate(
                        image, scale_factor=1.33, mode="bilinear", align_corners=False
                    )
                )
                probability.append(F.softmax(logit, -1))
            # --------------
            probability = torch.stack(probability, 0).mean(0)
        valid_num += batch_size
        valid_probability.append(probability.data.cpu().numpy())
        print(
            "\r %8d / %d  %s"
            % (
                valid_num,
                len(valid_loader.dataset),
                time_to_str(timer() - start_timer, "sec"),
            ),
            end="",
            flush=True,
        )
    assert valid_num == len(valid_loader.dataset)
    print("")
    probability = np.concatenate(valid_probability)
    return probability


class Logger(object):
    def __init__(self):
        self.terminal = sys.stdout  # stdout
        self.file = None

    def open(self, file, mode=None):
        if mode is None:
            mode = "w"
        self.file = open(file, mode)

    def write(self, message, is_terminal=1, is_file=1):
        if "\r" in message:
            is_file = 0
        if is_terminal == 1:
            self.terminal.write(message)
            self.terminal.flush()
            # time.sleep(1)
        if is_file == 1:
            self.file.write(message)
            self.file.flush()

    def flush(self):
        # this flush method is needed for python 3 compatibility.
        # this handles the flush command by doing nothing.
        # you might want to specify some extra behavior here.
        pass


def time_to_str(t, mode="min"):
    if mode == "min":
        t = int(t) / 60
        hr = t // 60
        min = t % 60
        return "%2d hr %02d min" % (hr, min)
    elif mode == "sec":
        t = int(t)
        min = t // 60
        sec = t % 60
        return "%2d min %02d sec" % (min, sec)
    else:
        raise NotImplementedError


def run_submit():
    for fold in [0, 1, 2, 3, 4]:
        out_dir = "./study_predict/"
        # initial_checkpoint = \
        #    _classify_model1_path + 'f' + str(fold) + '.pth'
        # v2
        initial_checkpoint = _classify_model2_path + "effv2-m-f" + str(fold) + ".pth"
        ## setup  ----------------------------------------
        # mode = 'local'
        mode = "remote"
        submit_dir = out_dir + "%s-fold%d" % (mode, fold)
        os.makedirs(submit_dir, exist_ok=True)
        log = Logger()
        log.open(out_dir + "log.submit.txt", mode="a")
        log.write("\n--- [START %s] %s\n\n" % (IDENTIFIER, "-" * 64))
        # log.write('\t%s\n' % COMMON_STRING)
        log.write("\n")
        #
        ## dataset ------------------------------------
        df_valid = make_fold("test")
        valid_dataset = SiimDataset(df_valid)
        valid_loader = DataLoader(
            valid_dataset,
            sampler=SequentialSampler(valid_dataset),
            batch_size=32,  # 128, #
            drop_last=False,
            num_workers=8,
            pin_memory=True,
            collate_fn=null_collate,
        )
        log.write("mode : %s\n" % (mode))
        log.write("valid_dataset : \n%s\n" % (valid_dataset))
        ## net ----------------------------------------
        if 1:
            net = NetV2().cuda()
            net.load_state_dict(
                torch.load(initial_checkpoint)["state_dict"], strict=True
            )
            # ---
            start_timer = timer()
            probability = do_predict(net, valid_loader)
            log.write("time %s \n" % time_to_str(timer() - start_timer, "min"))
            log.write("probability %s \n" % str(probability.shape))
            np.save(submit_dir + "/probability.npy", probability)
            df_valid.to_csv(submit_dir + "/df_valid.csv", index=False)
        else:
            probability = np.load(submit_dir + "/probability.npy")
        # ----
        df_study, df_image = probability_to_df_study(df_valid, probability)
        # df_image = probability_to_df_image(df_valid, None, None)
        # df_submit = pd.concat([df_study,df_image])
        df_submit = pd.concat([df_study])
        df_submit.to_csv(submit_dir + "/submit.csv", index=False)
        log.write("submit_dir : %s\n" % (submit_dir))
        log.write("initial_checkpoint : %s\n" % (initial_checkpoint))
        log.write("df_submit : %s\n" % str(df_submit.shape))
        log.write("%s\n" % str(df_submit))
        log.write("\n")


IDENTIFIER = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
run_submit()


# df_valid = make_fold('test')
# df_valid
def run_remote_ensemble():
    out_dir = "./study_predict/"
    log = Logger()
    log.open(out_dir + "log.submit.txt", mode="a")
    log.write("\n--- [START %s] %s\n\n" % (IDENTIFIER, "-" * 64))
    # log.write('\t%s\n' % COMMON_STRING)
    log.write("\n")
    submit_dir = [
        out_dir + "remote-fold0",
        out_dir + "remote-fold1",
        out_dir + "remote-fold2",
        out_dir + "remote-fold3",
        out_dir + "remote-fold4",
    ]
    probability = 0
    for d in submit_dir:
        p = np.load(d + "/probability.npy")
        probability += p**0.5
    probability = probability / len(submit_dir)
    # ----
    df_valid = pd.read_csv(submit_dir[0] + "/df_valid.csv")
    df_study, df_image = probability_to_df_study(df_valid, probability)
    # df_image  = probability_to_df_image(df_valid, None, None)
    # df_submit = pd.concat([df_study, df_image])
    df_submit = pd.concat([df_study])
    # df_submit.to_csv(out_dir + '/effb3-full-512-mask-submit-ensemble1.csv', index=False)
    log.write("submit_dir : %s\n" % (submit_dir))
    log.write("df_submit : %s\n" % str(df_submit.shape))
    log.write("%s\n" % str(df_submit))
    log.write("\n")
    return df_submit, df_image


IDENTIFIER = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
predict_study, df_image = run_remote_ensemble()
cuda.select_device(0)
cuda.close()
cuda.select_device(0)


# # Predict 2class
#!pip install /kaggle/input/kerasapplications -q
#!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps
# import efficientnet.tfkeras as efn
"""
import tensorflow as tf
def auto_select_accelerator():
    try:
        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
        tf.config.experimental_connect_to_cluster(tpu)
        tf.tpu.experimental.initialize_tpu_system(tpu)
        strategy = tf.distribute.experimental.TPUStrategy(tpu)
        print("Running on TPU:", tpu.master())
    except ValueError:
        strategy = tf.distribute.get_strategy()
    print(f"Running on {strategy.num_replicas_in_sync} replicas")
    return strategy
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
    def decode(path):
        file_bytes = tf.io.read_file(path)
        if ext == 'png':
            img = tf.image.decode_png(file_bytes, channels=3)
        elif ext in ['jpg', 'jpeg']:
            img = tf.image.decode_jpeg(file_bytes, channels=3)
        else:
            raise ValueError("Image extension not supported")
        img = tf.cast(img, tf.float32) / 255.0
        img = tf.image.resize(img, target_size)
        return img
    def decode_with_labels(path, label):
        return decode(path), label
    return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
    def augment(img):
        img = tf.image.random_flip_left_right(img)
        img = tf.image.random_flip_up_down(img)
        return img
    def augment_with_labels(img, label):
        return augment(img), label
    return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
                  decode_fn=None, augment_fn=None,
                  augment=True, repeat=True, shuffle=1024, 
                  cache_dir=""):
    if cache_dir != "" and cache is True:
        os.makedirs(cache_dir, exist_ok=True)
    if decode_fn is None:
        decode_fn = build_decoder(labels is not None)
    if augment_fn is None:
        augment_fn = build_augmenter(labels is not None)
    AUTO = tf.data.experimental.AUTOTUNE
    slices = paths if labels is None else (paths, labels)
    dset = tf.data.Dataset.from_tensor_slices(slices)
    dset = dset.map(decode_fn, num_parallel_calls=AUTO)
    dset = dset.cache(cache_dir) if cache else dset
    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset
    dset = dset.repeat() if repeat else dset
    dset = dset.shuffle(shuffle) if shuffle else dset
    dset = dset.batch(bsize).prefetch(AUTO)
    return dset
#COMPETITION_NAME = "siim-cov19-test-img512-study-600"
strategy = auto_select_accelerator()
BATCH_SIZE = strategy.num_replicas_in_sync * 16
IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 512)
"""
submit_df = pd.read_csv("/kaggle/input/siim-covid19-detection/sample_submission.csv")
df_2class = meta_df.loc[meta_df.split == "test"].copy()
# test_paths = f'/kaggle/input/covid19512/test/' + df_2class['image_id'] +'.png'
df_2class["none"] = 0
# label_cols = df_2class.columns[5]
# label_cols
# 注意！！！！！！ 这个模型训练时 没有 drop duplicate
"""
test_decoder = build_decoder(with_labels=False, target_size=(IMG_SIZE, IMG_SIZE), ext='png')
dtest = build_dataset(
    test_paths, bsize=BATCH_SIZE, repeat=False, 
    shuffle=False, augment=False, cache=False,
    decode_fn=test_decoder
)
with strategy.scope():
    models = []
    models0 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model0.h5'
    )
    models1 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model1.h5'
    )
    models2 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model2.h5'
    )
    models3 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model3.h5'
    )
    models4 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model4.h5'
    )
    models.append(models0)
    models.append(models1)
    models.append(models2)
    models.append(models3)
    models.append(models4)
df_2class[label_cols] = sum([model.predict(dtest, verbose=1) for model in models]) / len(models)
df_2class = df_2class.reset_index(drop=True)
"""
df_2class["none"] = df_image["negative"].values
df_2class.head()
# cuda.select_device(0)
# cuda.close()
# cuda.select_device(0)


# # Detect
shutil.copytree(
    "/kaggle/input/yolov5-official-v31-dataset/yolov5", "/kaggle/working/yolov5"
)
os.chdir("/kaggle/working/yolov5")
MODEL_PATH = (
    _det_model_path
    + "yolom-f0.pt"
    + " "
    + _det_model_path
    + "yolom-f1.pt"
    + " "
    + _det_model_path
    + "yolom-f2.pt"
    + " "
    + _det_model_path
    + "yolom-f3.pt"
    + " "
    + _det_model_path
    + "yolom-f4.pt"
)
"""
                  --source {_test_files_path} \
                  --img {IMG_SIZE} \
                  --conf 0.001 \
                  --iou-thres 0.5 \
                  --save-txt \
                  --save-conf
"""
"""
def yolo2voc(image_height, image_width, bboxes):
    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int
    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width
    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height
    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2
    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]
    return bboxes
image_ids = []
PredictionStrings = []
#for file_path in tqdm(glob('runs/detect/exp/labels/*.txt')):
for dir_path, _, filenames in os.walk(_test_files_path):
        print(len(filenames))
for file in filenames:
    file_path = 'runs/detect/exp/labels/' + file.replace(".png", '.txt')
    image_id = file_path.split('/')[-1].split('.')[0]
    w, h = meta_df.loc[meta_df.image_id == image_id,['dim1', 'dim0']].values[0]
    if not os.path.exists(file_path):
        bboxes = "none 1 0 0 1 1"
    else:
        f = open(file_path, 'r')
        data = np.array(f.read().replace('\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)
        data = data[:, [0, 5, 1, 2, 3, 4]]
        bboxes = list(np.round(np.concatenate((data[:, :2], np.round(yolo2voc(h, w, data[:, 2:]))), axis =1).reshape(-1), 12).astype(str))
        for idx in range(len(bboxes)):
            bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]
        bboxes = ' '.join(bboxes)
    image_id += "_image"
    image_ids.append(image_id)
    PredictionStrings.append(bboxes)
predict_image = pd.DataFrame({'id':image_ids,
                        'PredictionString':PredictionStrings})
"""


# # Submit
"""
for i in range(predict_image.shape[0]):
    if predict_image.loc[i,'PredictionString'] == "none 1 0 0 1 1":
        continue
    sub_df_split = predict_image.loc[i,'PredictionString'].split()
    sub_df_list = []
    for j in range(int(len(sub_df_split) / 6)):
        sub_df_list.append('opacity')
        sub_df_list.append(sub_df_split[6 * j + 1])
        sub_df_list.append(sub_df_split[6 * j + 2])
        sub_df_list.append(sub_df_split[6 * j + 3])
        sub_df_list.append(sub_df_split[6 * j + 4])
        sub_df_list.append(sub_df_split[6 * j + 5])
    predict_image.loc[i,'PredictionString'] = ' '.join(sub_df_list)
"""
"""
for i in range(predict_image.shape[0]):
    if predict_image.loc[i,'PredictionString'] != 'none 1 0 0 1 1':
        _none = str(df_2class.loc[df_2class.image_id + "_image" == predict_image.iloc[i].id]['none'].item())
        predict_image.loc[i,'PredictionString'] = predict_image.loc[i,'PredictionString'] + ' none ' + _none + ' 0 0 1 1'
"""
for index, row in predict_study.iterrows():
    submit_df.loc[submit_df.id == row.id, "PredictionString"] = row.PredictionString
# for index, row in predict_image.iterrows():
#    submit_df.loc[submit_df.id == row.id, "PredictionString"] = row.PredictionString
submit_df.to_csv("/kaggle/working/submission.csv", index=False)
"""
sample = submit_df.iloc[-1]
print(sample.id, sample.PredictionString)
_study = meta_df.loc[meta_df.image_id + "_image" == sample.id].study.item()
print(_study)
_study_item = submit_df.loc[submit_df.id == _study + "_study"]
_study_item
"""
shutil.rmtree("/kaggle/working/yolov5")
shutil.rmtree("/kaggle/working/study_predict")
<|endoftext|>
[{"Id": 70254580, "ScriptId": 19223412, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 4321793, "CreationDate": "08/05/2021 14:26:20", "VersionNumber": 2.0, "Title": "v2 covid-submit", "EvaluationDate": "08/05/2021", "IsChange": true, "TotalLines": 804.0, "LinesInsertedFromPrevious": 1.0, "LinesChangedFromPrevious": 0.0, "LinesUnchangedFromPrevious": 803.0, "LinesInsertedFromFork": 109.0, "LinesDeletedFromFork": 16.0, "LinesChangedFromFork": 0.0, "LinesUnchangedFromFork": 695.0, "TotalVotes": 0}]
[{"Id": 93595129, "KernelVersionId": 70254580, "SourceDatasetVersionId": 2411714}, {"Id": 93595128, "KernelVersionId": 70254580, "SourceDatasetVersionId": 1800778}, {"Id": 93595130, "KernelVersionId": 70254580, "SourceDatasetVersionId": 2417566}]
[{"Id": 2411714, "DatasetId": 1454561, "DatasourceVersionId": 2453830, "CreatorUserId": 4321793, "LicenseName": "Unknown", "CreationDate": "07/10/2021 07:49:01", "VersionNumber": 2.0, "Title": "covid19-512", "Slug": "covid19512", "Subtitle": NaN, "Description": NaN, "VersionNotes": "\u66f4\u65b0mate", "TotalCompressedBytes": 0.0, "TotalUncompressedBytes": 0.0}]
[{"Id": 1454561, "CreatorUserId": 4321793, "OwnerUserId": 4321793.0, "OwnerOrganizationId": NaN, "CurrentDatasetVersionId": 2411714.0, "CurrentDatasourceVersionId": 2453830.0, "ForumId": 1474127, "Type": 2, "CreationDate": "07/08/2021 02:43:21", "LastActivityDate": "07/08/2021", "TotalViews": 860, "TotalDownloads": 309, "TotalVotes": 1, "TotalKernels": 4}]
# # Warrning
# 1. This notebook only include public testset for faster testif you submit it you will get 0 score
# 2. Change wandb API KEY in training notebook to yours
# 3. colab free GPU is about 3x faster than kaggle
# # Approach and Refferences
# > efficientnetb3a with aux loss for study + efficientnetb5 for 2class + yolov5m for image
# ## Refferences
# 1. henhttps://www.kaggle.com/c/siim-covid19-detection/discussion/246586gk's aux loss https://www.kaggle.com/c/siim-covid19-detection/discussion/240233
# 2. alien's 2 class tricks https://www.kaggle.com/c/siim-covid19-detection/discussion/246586
# 3. darian's duplicate analysis https://www.kaggle.com/c/siim-covid19-detection/discussion/240878
# ## Training notebook
# 1. drop duplicate and create mask https://www.kaggle.com/drzhuzhe/siiim-covid-stratified-k-fold-and-create-mask
# 2. training study level https://www.kaggle.com/drzhuzhe/covid19-classify efficientnetb3a 1e-3 10 ep + 1e-4 5ep CV map*0.66 score (3.76 + 3.92 + 3.85 + 3.7 + 3.6)/5 avg 3.766
# 3. training yolo https://www.kaggle.com/drzhuzhe/covid19-det?scriptVersionId=67605495 efficientnetb5 only 10 epoch  CV map score (0.4947 + 0.5103 + 0.4848 +0.4692 +0.5198)/5 avg 0.49575
# 4. 2 class https://www.kaggle.com/drzhuzhe/siim-covid19-efnb7-train-fold0-5-2class  yolov5m 15 epoch (0.869 + 0.860 + 0.882 + 0.878 + 0.876)/5 avg 0.872
# ## Experiments
# 1. study level with 6 class is doable single fold LB 0.586 +
# 2. efficientnetV2-m got low result https://www.kaggle.com/c/siim-covid19-detection/discussion/248442 according to this disscus efficientnetV2 may need larger batchsize
# 3. aux CNN head attach to block 4 get mediocre result
#
import numpy as np, pandas as pd
from glob import glob
import shutil, os
import matplotlib.pyplot as plt
from sklearn.model_selection import GroupKFold
from tqdm.notebook import tqdm
import seaborn as sns
import cv2
import sys
import math
from timeit import default_timer as timer
from datetime import datetime
from numba import cuda

sys.path.append("../input/timm-pytorch-image-models/pytorch-image-models-master")
_det_model_path = "/kaggle/input/collect-submit-model/det/"
_classify_model1_path = "/kaggle/input/covidmodels/Archive/classify-ep12/"
_classify_model2_path = "/kaggle/input/covidmodels/"
_test_files_path = "/kaggle/input/covid19512/test/"
_data_dir = "/kaggle/input/covid19512/"
meta_df = pd.read_csv(_data_dir + "meta.csv")
IMG_SIZE = 512

# # Clssification
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.parallel.data_parallel import data_parallel
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader
from torch.utils.data.sampler import *
import collections
from collections import defaultdict
import timm
from timm.models.efficientnet import *
import torch.cuda.amp as amp

data_dir = _data_dir
image_size = IMG_SIZE
study_name_to_predict_string = {
    "Negative for Pneumonia": "negative",
    "Typical Appearance": "typical",
    "Indeterminate Appearance": "indeterminate",
    "Atypical Appearance": "atypical",
}
study_name_to_label = {
    "Negative for Pneumonia": 0,
    "Typical Appearance": 1,
    "Indeterminate Appearance": 2,
    "Atypical Appearance": 3,
}
study_label_to_name = {v: k for k, v in study_name_to_label.items()}
num_study_label = len(study_name_to_label)


def make_fold(mode="train-0"):
    if "test" in mode:
        df_meta = pd.read_csv(data_dir + "meta.csv")
        df_valid = df_meta[df_meta["split"] == "test"].copy()
        for l in study_name_to_label.keys():
            df_valid.loc[:, l] = 0
        df_valid = df_valid.reset_index(drop=True)
        return df_valid


class SiimDataset(Dataset):
    def __init__(self, df, augment=None):
        super().__init__()
        self.df = df
        self.augment = augment
        self.length = len(df)

    def __str__(self):
        string = ""
        string += "\tlen = %d\n" % len(self)
        string += "\tdf  = %s\n" % str(self.df.shape)
        string += "\tlabel distribution\n"
        for i in range(num_study_label):
            n = self.df[study_label_to_name[i]].sum()
            string += "\t\t %d %26s: %5d (%0.4f)\n" % (
                i,
                study_label_to_name[i],
                n,
                n / len(self.df),
            )
        return string

    def __len__(self):
        return self.length

    def __getitem__(self, index):
        d = self.df.iloc[index]
        image_file = data_dir + "/test/%s.png" % (d.image_id)
        image = cv2.imread(image_file, cv2.IMREAD_GRAYSCALE)
        onehot = d[study_name_to_label.keys()].values
        mask = np.zeros_like(image)
        r = {
            "index": index,
            "d": d,
            "image": image,
            "mask": mask,
            "onehot": onehot,
        }
        if self.augment is not None:
            r = self.augment(r)
        return r


def null_collate(batch):
    collate = defaultdict(list)
    for r in batch:
        for k, v in r.items():
            collate[k].append(v)
    # ---
    batch_size = len(batch)
    onehot = np.ascontiguousarray(np.stack(collate["onehot"])).astype(np.float32)
    collate["onehot"] = torch.from_numpy(onehot)
    image = np.stack(collate["image"])
    image = image.reshape(batch_size, 1, image_size, image_size).repeat(3, 1)
    image = np.ascontiguousarray(image)
    image = image.astype(np.float32) / 255
    collate["image"] = torch.from_numpy(image)
    mask = np.stack(collate["mask"])
    mask = mask.reshape(batch_size, 1, image_size, image_size)
    mask = np.ascontiguousarray(mask)
    mask = mask.astype(np.float32) / 255
    collate["mask"] = torch.from_numpy(mask)
    return collate


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        e = efficientnet_b3a(pretrained=False, drop_rate=0.3, drop_path_rate=0.2)
        self.b0 = nn.Sequential(
            e.conv_stem,
            e.bn1,
            e.act1,
        )
        self.b1 = e.blocks[0]
        self.b2 = e.blocks[1]
        self.b3 = e.blocks[2]
        self.b4 = e.blocks[3]
        self.b5 = e.blocks[4]
        self.b6 = e.blocks[5]
        self.b7 = e.blocks[6]
        self.b8 = nn.Sequential(
            e.conv_head,  # 384, 1536
            e.bn2,
            e.act2,
        )
        self.logit = nn.Linear(1536, num_study_label)
        self.mask = nn.Sequential(
            nn.Conv2d(136, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 1, kernel_size=1, padding=0),
        )

    # @torch.cuda.amp.autocast()
    def forward(self, image):
        batch_size = len(image)
        x = 2 * image - 1  # ; print('input ',   x.shape)
        x = self.b0(x)  # ; print (x.shape)  # torch.Size([2, 40, 256, 256])
        x = self.b1(x)  # ; print (x.shape)  # torch.Size([2, 24, 256, 256])
        x = self.b2(x)  # ; print (x.shape)  # torch.Size([2, 32, 128, 128])
        x = self.b3(x)  # ; print (x.shape)  # torch.Size([2, 48, 64, 64])
        x = self.b4(x)  # ; print (x.shape)  # torch.Size([2, 96, 32, 32])
        x = self.b5(x)  # ; print (x.shape)  # torch.Size([2, 136, 32, 32])
        # ------------
        mask = self.mask(x)
        # -------------
        x = self.b6(x)  # ; print (x.shape)  # torch.Size([2, 232, 16, 16])
        x = self.b7(x)  # ; print (x.shape)  # torch.Size([2, 384, 16, 16])
        x = self.b8(x)  # ; print (x.shape)  # torch.Size([2, 1536, 16, 16])
        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)
        # x = F.dropout(x, 0.5, training=self.training)
        logit = self.logit(x)
        return logit, mask


class NetV2(nn.Module):
    def __init__(self):
        super(NetV2, self).__init__()
        # e = efficientnet_b3a(pretrained=True, drop_rate=0.3, drop_path_rate=0.2)
        # e = efficientnetv2_rw_m(pretrained=True, drop_rate=0.5, drop_path_rate=0.2)
        # e = tf_efficientnetv2_m_in21ft1k(pretrained=True, drop_rate=0.3, drop_path_rate=0.2)
        e = tf_efficientnetv2_m(pretrained=False, drop_path_rate=0.4)
        self.b0 = nn.Sequential(
            e.conv_stem,
            e.bn1,
            e.act1,
        )
        self.b1 = e.blocks[0]
        self.b2 = e.blocks[1]
        self.b3 = e.blocks[2]
        self.b4 = e.blocks[3]
        self.b5 = e.blocks[4]
        self.b6 = e.blocks[5]
        self.b7 = e.blocks[6]
        self.b8 = nn.Sequential(
            e.conv_head,  # 384, 1536
            e.bn2,
            e.act2,
        )
        self.logit = nn.Linear(1280, num_study_label)
        self.mask = nn.Sequential(
            nn.Conv2d(176, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            # nn.Conv2d(128, 128, kernel_size=3, padding=1),
            # nn.BatchNorm2d(128),
            # nn.ReLU(inplace=True),
            nn.Conv2d(128, 1, kernel_size=1, padding=0),
        )

    # @torch.cuda.amp.autocast()
    def forward(self, image):
        batch_size = len(image)
        x = 2 * image - 1  # ; print('input ',   x.shape)
        """
        rw_m
        torch.Size([2, 32, 256, 256])
        torch.Size([2, 32, 256, 256])
        torch.Size([2, 56, 128, 128])
        torch.Size([2, 80, 64, 64])
        torch.Size([2, 152, 32, 32])
        torch.Size([2, 192, 32, 32])
        torch.Size([2, 328, 16, 16])
        torch.Size([2, 2152, 16, 16])
        """
        x = self.b0(x)
        # print (x.shape)  # torch.Size([2, 32, 256, 256])
        x = self.b1(x)
        # print (x.shape)  # torch.Size([2, 32, 256, 256])
        x = self.b2(x)
        # print (x.shape)  # torch.Size([2, 56, 128, 128])
        x = self.b3(x)
        # print (x.shape)  # torch.Size([2, 80, 64, 64])
        x = self.b4(x)
        # print (x.shape)  # torch.Size([2, 152, 32, 32])
        x = self.b5(x)
        # print (x.shape)  # torch.Size([2, 192, 32, 32])
        # ------------
        mask = self.mask(x)
        # -------------
        x = self.b6(x)
        # print (x.shape)  # torch.Size([2, 328, 16, 16])
        x = self.b7(x)
        # print (x.shape)  # torch.Size([2, 512, 16, 16])
        x = self.b8(x)
        # print (x.shape)  # torch.Size([2, 2152, 16, 16])
        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)
        x = F.dropout(x, 0.5, training=self.training)
        logit = self.logit(x)
        return logit, mask


def probability_to_df_study(df_valid, probability):
    df_study = pd.DataFrame()
    df_image = df_valid.copy()
    df_study.loc[:, "id"] = df_valid.study + "_study"
    for i in range(num_study_label):
        df_study.loc[
            :, study_name_to_predict_string[study_label_to_name[i]]
        ] = probability[:, i]
        df_image.loc[
            :, study_name_to_predict_string[study_label_to_name[i]]
        ] = probability[:, i]
    df_study = df_study.groupby("id", as_index=False).mean()
    df_study.loc[:, "PredictionString"] = (
        "negative "
        + df_study.negative.apply(lambda x: "%0.6f" % x)
        + " 0 0 1 1"
        + " typical "
        + df_study.typical.apply(lambda x: "%0.6f" % x)
        + " 0 0 1 1"
        + " indeterminate "
        + df_study.indeterminate.apply(lambda x: "%0.6f" % x)
        + " 0 0 1 1"
        + " atypical "
        + df_study.atypical.apply(lambda x: "%0.6f" % x)
        + " 0 0 1 1"
    )
    df_study = df_study[["id", "PredictionString"]]
    return df_study, df_image


def do_predict(net, valid_loader, tta=["flip", "scale"]):  # flip
    valid_probability = []
    valid_num = 0
    start_timer = timer()
    for t, batch in enumerate(valid_loader):
        batch_size = len(batch["index"])
        image = batch["image"].cuda()
        onehot = batch["onehot"]
        label = onehot.argmax(-1)
        # <todo> TTA
        net.eval()
        with torch.no_grad():
            probability = []
            logit, mask = net(image)
            probability.append(F.softmax(logit, -1))
            if "flip" in tta:
                logit, mask = net(torch.flip(image, dims=(3,)))
                probability.append(F.softmax(logit, -1))
            if "scale" in tta:
                # size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None):
                logit, mask = net(
                    F.interpolate(
                        image, scale_factor=1.33, mode="bilinear", align_corners=False
                    )
                )
                probability.append(F.softmax(logit, -1))
            # --------------
            probability = torch.stack(probability, 0).mean(0)
        valid_num += batch_size
        valid_probability.append(probability.data.cpu().numpy())
        print(
            "\r %8d / %d  %s"
            % (
                valid_num,
                len(valid_loader.dataset),
                time_to_str(timer() - start_timer, "sec"),
            ),
            end="",
            flush=True,
        )
    assert valid_num == len(valid_loader.dataset)
    print("")
    probability = np.concatenate(valid_probability)
    return probability


class Logger(object):
    def __init__(self):
        self.terminal = sys.stdout  # stdout
        self.file = None

    def open(self, file, mode=None):
        if mode is None:
            mode = "w"
        self.file = open(file, mode)

    def write(self, message, is_terminal=1, is_file=1):
        if "\r" in message:
            is_file = 0
        if is_terminal == 1:
            self.terminal.write(message)
            self.terminal.flush()
            # time.sleep(1)
        if is_file == 1:
            self.file.write(message)
            self.file.flush()

    def flush(self):
        # this flush method is needed for python 3 compatibility.
        # this handles the flush command by doing nothing.
        # you might want to specify some extra behavior here.
        pass


def time_to_str(t, mode="min"):
    if mode == "min":
        t = int(t) / 60
        hr = t // 60
        min = t % 60
        return "%2d hr %02d min" % (hr, min)
    elif mode == "sec":
        t = int(t)
        min = t // 60
        sec = t % 60
        return "%2d min %02d sec" % (min, sec)
    else:
        raise NotImplementedError


def run_submit():
    for fold in [0, 1, 2, 3, 4]:
        out_dir = "./study_predict/"
        # initial_checkpoint = \
        #    _classify_model1_path + 'f' + str(fold) + '.pth'
        # v2
        initial_checkpoint = _classify_model2_path + "effv2-m-f" + str(fold) + ".pth"
        ## setup  ----------------------------------------
        # mode = 'local'
        mode = "remote"
        submit_dir = out_dir + "%s-fold%d" % (mode, fold)
        os.makedirs(submit_dir, exist_ok=True)
        log = Logger()
        log.open(out_dir + "log.submit.txt", mode="a")
        log.write("\n--- [START %s] %s\n\n" % (IDENTIFIER, "-" * 64))
        # log.write('\t%s\n' % COMMON_STRING)
        log.write("\n")
        #
        ## dataset ------------------------------------
        df_valid = make_fold("test")
        valid_dataset = SiimDataset(df_valid)
        valid_loader = DataLoader(
            valid_dataset,
            sampler=SequentialSampler(valid_dataset),
            batch_size=32,  # 128, #
            drop_last=False,
            num_workers=8,
            pin_memory=True,
            collate_fn=null_collate,
        )
        log.write("mode : %s\n" % (mode))
        log.write("valid_dataset : \n%s\n" % (valid_dataset))
        ## net ----------------------------------------
        if 1:
            net = NetV2().cuda()
            net.load_state_dict(
                torch.load(initial_checkpoint)["state_dict"], strict=True
            )
            # ---
            start_timer = timer()
            probability = do_predict(net, valid_loader)
            log.write("time %s \n" % time_to_str(timer() - start_timer, "min"))
            log.write("probability %s \n" % str(probability.shape))
            np.save(submit_dir + "/probability.npy", probability)
            df_valid.to_csv(submit_dir + "/df_valid.csv", index=False)
        else:
            probability = np.load(submit_dir + "/probability.npy")
        # ----
        df_study, df_image = probability_to_df_study(df_valid, probability)
        # df_image = probability_to_df_image(df_valid, None, None)
        # df_submit = pd.concat([df_study,df_image])
        df_submit = pd.concat([df_study])
        df_submit.to_csv(submit_dir + "/submit.csv", index=False)
        log.write("submit_dir : %s\n" % (submit_dir))
        log.write("initial_checkpoint : %s\n" % (initial_checkpoint))
        log.write("df_submit : %s\n" % str(df_submit.shape))
        log.write("%s\n" % str(df_submit))
        log.write("\n")


IDENTIFIER = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
run_submit()


# df_valid = make_fold('test')
# df_valid
def run_remote_ensemble():
    out_dir = "./study_predict/"
    log = Logger()
    log.open(out_dir + "log.submit.txt", mode="a")
    log.write("\n--- [START %s] %s\n\n" % (IDENTIFIER, "-" * 64))
    # log.write('\t%s\n' % COMMON_STRING)
    log.write("\n")
    submit_dir = [
        out_dir + "remote-fold0",
        out_dir + "remote-fold1",
        out_dir + "remote-fold2",
        out_dir + "remote-fold3",
        out_dir + "remote-fold4",
    ]
    probability = 0
    for d in submit_dir:
        p = np.load(d + "/probability.npy")
        probability += p**0.5
    probability = probability / len(submit_dir)
    # ----
    df_valid = pd.read_csv(submit_dir[0] + "/df_valid.csv")
    df_study, df_image = probability_to_df_study(df_valid, probability)
    # df_image  = probability_to_df_image(df_valid, None, None)
    # df_submit = pd.concat([df_study, df_image])
    df_submit = pd.concat([df_study])
    # df_submit.to_csv(out_dir + '/effb3-full-512-mask-submit-ensemble1.csv', index=False)
    log.write("submit_dir : %s\n" % (submit_dir))
    log.write("df_submit : %s\n" % str(df_submit.shape))
    log.write("%s\n" % str(df_submit))
    log.write("\n")
    return df_submit, df_image


IDENTIFIER = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
predict_study, df_image = run_remote_ensemble()
cuda.select_device(0)
cuda.close()
cuda.select_device(0)


# # Predict 2class
#!pip install /kaggle/input/kerasapplications -q
#!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps
# import efficientnet.tfkeras as efn
"""
import tensorflow as tf
def auto_select_accelerator():
    try:
        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
        tf.config.experimental_connect_to_cluster(tpu)
        tf.tpu.experimental.initialize_tpu_system(tpu)
        strategy = tf.distribute.experimental.TPUStrategy(tpu)
        print("Running on TPU:", tpu.master())
    except ValueError:
        strategy = tf.distribute.get_strategy()
    print(f"Running on {strategy.num_replicas_in_sync} replicas")
    return strategy
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
    def decode(path):
        file_bytes = tf.io.read_file(path)
        if ext == 'png':
            img = tf.image.decode_png(file_bytes, channels=3)
        elif ext in ['jpg', 'jpeg']:
            img = tf.image.decode_jpeg(file_bytes, channels=3)
        else:
            raise ValueError("Image extension not supported")
        img = tf.cast(img, tf.float32) / 255.0
        img = tf.image.resize(img, target_size)
        return img
    def decode_with_labels(path, label):
        return decode(path), label
    return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
    def augment(img):
        img = tf.image.random_flip_left_right(img)
        img = tf.image.random_flip_up_down(img)
        return img
    def augment_with_labels(img, label):
        return augment(img), label
    return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
                  decode_fn=None, augment_fn=None,
                  augment=True, repeat=True, shuffle=1024, 
                  cache_dir=""):
    if cache_dir != "" and cache is True:
        os.makedirs(cache_dir, exist_ok=True)
    if decode_fn is None:
        decode_fn = build_decoder(labels is not None)
    if augment_fn is None:
        augment_fn = build_augmenter(labels is not None)
    AUTO = tf.data.experimental.AUTOTUNE
    slices = paths if labels is None else (paths, labels)
    dset = tf.data.Dataset.from_tensor_slices(slices)
    dset = dset.map(decode_fn, num_parallel_calls=AUTO)
    dset = dset.cache(cache_dir) if cache else dset
    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset
    dset = dset.repeat() if repeat else dset
    dset = dset.shuffle(shuffle) if shuffle else dset
    dset = dset.batch(bsize).prefetch(AUTO)
    return dset
#COMPETITION_NAME = "siim-cov19-test-img512-study-600"
strategy = auto_select_accelerator()
BATCH_SIZE = strategy.num_replicas_in_sync * 16
IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 512)
"""
submit_df = pd.read_csv("/kaggle/input/siim-covid19-detection/sample_submission.csv")
df_2class = meta_df.loc[meta_df.split == "test"].copy()
# test_paths = f'/kaggle/input/covid19512/test/' + df_2class['image_id'] +'.png'
df_2class["none"] = 0
# label_cols = df_2class.columns[5]
# label_cols
# 注意！！！！！！ 这个模型训练时 没有 drop duplicate
"""
test_decoder = build_decoder(with_labels=False, target_size=(IMG_SIZE, IMG_SIZE), ext='png')
dtest = build_dataset(
    test_paths, bsize=BATCH_SIZE, repeat=False, 
    shuffle=False, augment=False, cache=False,
    decode_fn=test_decoder
)
with strategy.scope():
    models = []
    models0 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model0.h5'
    )
    models1 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model1.h5'
    )
    models2 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model2.h5'
    )
    models3 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model3.h5'
    )
    models4 = tf.keras.models.load_model(
        '../input/siim-covid19-efnb7-train-fold0-5-2class/model4.h5'
    )
    models.append(models0)
    models.append(models1)
    models.append(models2)
    models.append(models3)
    models.append(models4)
df_2class[label_cols] = sum([model.predict(dtest, verbose=1) for model in models]) / len(models)
df_2class = df_2class.reset_index(drop=True)
"""
df_2class["none"] = df_image["negative"].values
df_2class.head()
# cuda.select_device(0)
# cuda.close()
# cuda.select_device(0)


# # Detect
shutil.copytree(
    "/kaggle/input/yolov5-official-v31-dataset/yolov5", "/kaggle/working/yolov5"
)
os.chdir("/kaggle/working/yolov5")
MODEL_PATH = (
    _det_model_path
    + "yolom-f0.pt"
    + " "
    + _det_model_path
    + "yolom-f1.pt"
    + " "
    + _det_model_path
    + "yolom-f2.pt"
    + " "
    + _det_model_path
    + "yolom-f3.pt"
    + " "
    + _det_model_path
    + "yolom-f4.pt"
)
"""
                  --source {_test_files_path} \
                  --img {IMG_SIZE} \
                  --conf 0.001 \
                  --iou-thres 0.5 \
                  --save-txt \
                  --save-conf
"""
"""
def yolo2voc(image_height, image_width, bboxes):
    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int
    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width
    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height
    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2
    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]
    return bboxes
image_ids = []
PredictionStrings = []
#for file_path in tqdm(glob('runs/detect/exp/labels/*.txt')):
for dir_path, _, filenames in os.walk(_test_files_path):
        print(len(filenames))
for file in filenames:
    file_path = 'runs/detect/exp/labels/' + file.replace(".png", '.txt')
    image_id = file_path.split('/')[-1].split('.')[0]
    w, h = meta_df.loc[meta_df.image_id == image_id,['dim1', 'dim0']].values[0]
    if not os.path.exists(file_path):
        bboxes = "none 1 0 0 1 1"
    else:
        f = open(file_path, 'r')
        data = np.array(f.read().replace('\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)
        data = data[:, [0, 5, 1, 2, 3, 4]]
        bboxes = list(np.round(np.concatenate((data[:, :2], np.round(yolo2voc(h, w, data[:, 2:]))), axis =1).reshape(-1), 12).astype(str))
        for idx in range(len(bboxes)):
            bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]
        bboxes = ' '.join(bboxes)
    image_id += "_image"
    image_ids.append(image_id)
    PredictionStrings.append(bboxes)
predict_image = pd.DataFrame({'id':image_ids,
                        'PredictionString':PredictionStrings})
"""


# # Submit
"""
for i in range(predict_image.shape[0]):
    if predict_image.loc[i,'PredictionString'] == "none 1 0 0 1 1":
        continue
    sub_df_split = predict_image.loc[i,'PredictionString'].split()
    sub_df_list = []
    for j in range(int(len(sub_df_split) / 6)):
        sub_df_list.append('opacity')
        sub_df_list.append(sub_df_split[6 * j + 1])
        sub_df_list.append(sub_df_split[6 * j + 2])
        sub_df_list.append(sub_df_split[6 * j + 3])
        sub_df_list.append(sub_df_split[6 * j + 4])
        sub_df_list.append(sub_df_split[6 * j + 5])
    predict_image.loc[i,'PredictionString'] = ' '.join(sub_df_list)
"""
"""
for i in range(predict_image.shape[0]):
    if predict_image.loc[i,'PredictionString'] != 'none 1 0 0 1 1':
        _none = str(df_2class.loc[df_2class.image_id + "_image" == predict_image.iloc[i].id]['none'].item())
        predict_image.loc[i,'PredictionString'] = predict_image.loc[i,'PredictionString'] + ' none ' + _none + ' 0 0 1 1'
"""
for index, row in predict_study.iterrows():
    submit_df.loc[submit_df.id == row.id, "PredictionString"] = row.PredictionString
# for index, row in predict_image.iterrows():
#    submit_df.loc[submit_df.id == row.id, "PredictionString"] = row.PredictionString
submit_df.to_csv("/kaggle/working/submission.csv", index=False)
"""
sample = submit_df.iloc[-1]
print(sample.id, sample.PredictionString)
_study = meta_df.loc[meta_df.image_id + "_image" == sample.id].study.item()
print(_study)
_study_item = submit_df.loc[submit_df.id == _study + "_study"]
_study_item
"""
shutil.rmtree("/kaggle/working/yolov5")
shutil.rmtree("/kaggle/working/study_predict")
