[NeurIPS 2023] LMC: Large Model Collaboration with Cross-assessment for Training-Free Open-Set Object Recognition
from torch.utils.data import DataLoader, Dataset
import numpy as np
import os, json
from torchvision.datasets import CIFAR10, CIFAR100
from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize, ToPILImage
from PIL import Image
from torchvision.datasets import ImageFolder
import glob
from torch.utils.data import Subset
try:
    from dataloaders.split import splits_2020
except:
    from split import splits_2020


def tiny_single_isolated_class_dino_loader(root="data/dalle_generated/union_set",
    tiny_dino_labels=['African_elephant', 'American_lobster', 'Christmas_stocking', 'European_fire_salamander', 'Labrador_retriever', 'Persian_cat', 'abacus', 'academic_gown', 'accordion', 'acorn', 'airplane', 'altar', 'ambulance', 'anemone', 'apple', 'arch_bridge', 'barn', 'barnacles', 'barrel', 'basketball', 'beaker', 'bee', 'beetle', 'bell_pepper', 'bighorn', 'binoculars', 'birdhouse', 'bison', 'black_stork', 'blender', 'book', 'bowling_ball', 'brain_coral', 'brass', 'brown_bear', 'bucket', 'bullet_train', 'burger', 'butcher_shop', 'cable-stayed_bridge', 'cable_car', 'cactus', 'canyon', 'cappuccino', 'cardigan', 'carriage', 'carrot', 'cart', 'castanets', 'cat', 'cattle_prod', 'chameleon', 'cheeseburger', 'chest', 'chicken_coop', 'choir_robe', 'christmas_tree', 'church', 'cliff_dwelling', 'clock', 'coconut', 'coffee_mug', 'coffee_table', 'computer_keyboard', 'confectionery', 'console_table', 'copper', 'coral_reef', 'corkscrew', 'cowboy_hat', 'crane', 'crate', 'cruise_ship', 'cucumber', 'cuttlefish', 'daisy', 'dandelion', 'digital_clock', 'dining_chair', 'dining_table', 'dobro', 'dog', 'dolphin', 'dragonfly', 'drumstick', 'dugong', 'dumbbell', 'dumbwaiter', 'eagle', 'elephant', 'envelope', 'espresso', 'fire_hydrant', 'fire_truck', 'flagstaff', 'flamingo', 'fleece', 'flip-flop', 'fly', 'flying_saucer', 'fountain', 'fox', 'frisbee', 'funicular', 'giraffe', 'gloves', 'golden_retriever', 'goldfish', 'golf_ball', 'gondola', 'goose', 'grasshopper', 'grizzly_bear', 'guitar', 'handcuffs', 'harp', 'hat', 'hay_bale', 'head_cabbage', 'hedge_trimmer', 'helicopter', 'hill', 'holly', 'horse', 'hourglass', 'jellyfish', 'jigsaw_puzzle', 'judge_robe', 'kangaroo', 'kayak', 'king_penguin', 'ladybug', 'lakeside', 'lamp', 'latte', 'lawn_mower', 'lemon', 'lesser_panda', 'letter_opener', 'lion', 'lipstick', 'macchiato', 'mailbox', 'manatee', 'mandolin', 'mantis', 'measuring_spoon', 'meat_loaf', 'megaphone', 'microscope', 'military_uniform', 'milk_can', 'monarch', 'mongoose', 'monorail', 'mosque', 'moss', 'motorbike', 'motorcycle', 'mountain', 'mountain_bike', 'neck_brace', 'nurse_uniform', 'ocarina', 'ocean', 'octopus', 'oil_lamp', 'orange', 'oscilloscope', 'otter', 'paintbrush', 'pancake', 'papaya', 'peach', 'peacock', 'periscope', 'piano_bench', 'pillow', 'pinecone', 'pipe_wrench', 'pizza', 'plastic_bag', 'plate', 'police_van', 'pomegranate', 'poncho', 'popsicle_stick', 'power_drill', 'punching_bag', 'quesadilla', 'red_panda', 'refrigerator', 'river', 'rooster', 'rugby_ball', 'saddlebag', 'sailboat', 'sandal', 'sandwich', 'saxophone', 'scarf', 'school_bus', 'scorpion', 'scuba_diver', 'sea_anemone', 'sea_cucumber', 'sea_slug', 'sea_urchin', 'seahorse', 'seashore', 'seaweed', 'seesaw', 'sewing_machine', 'sewing_needle', 'shawl', 'shower_cap', 'side_table', 'silverware', 'sleeping_bag', 'snail', 'snowmobile', 'soccer_ball', 'spider_web', 'spiny_lobster', 'sports_car', 'standard_poodle', 'starfish', 'stingray', 'stopwatch', 'street_sign', 'subway_car', 'subway_train', 'sushi_roll', 'suspension_bridge', 'sweater', 'swimming_trunks', 'swordfish', 'table_lamp', 'tape_player', 'teddy', 'telescope', 'temple', 'tennis_ball', 'thatch', 'thatched_roof', 'thimble', 'tiger', 'toaster', 'tomato', 'tortilla', 'toy_poodle', 'traffic_light', 'tram', 'trash_can', 'tree_frog', 'trilobite', 'trolleybus', 'trophy', 'trowel', 'truss_bridge', 'turnstile', 'umbrella', 'vacuum_cleaner', 'vase', 'vestment', 'viaduct', 'violin_case', 'volleyball', 'washbasin', 'water_tower', 'waterfall', 'watering_can', 'watermelon', 'whale', 'whale_shark', 'wheelbarrow', 'window_screen', 'wine_barrel', 'wok', 'wolf', 'wooden_crate', 'wooden_spoon']
                                           ):
    loaders_dict = {}
    transform = Compose([
        Resize(224, interpolation=3),
        CenterCrop(224),
        ToTensor(),
        Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])
    for class_label in tiny_dino_labels:
       dataset = ImageFolder(root=root, transform=transform)
       idx = [i for i in range(len(dataset)) if dataset.imgs[i][1] == dataset.class_to_idx[class_label]]
       # build the appropriate subset
       subset = Subset(dataset, idx)
       loader = DataLoader(dataset=subset, batch_size=1, num_workers=4)
       loaders_dict[class_label] = loader
    return loaders_dict, tiny_dino_labels

def tinyimage_get_all_labels():
    dataset = ImageFolder(root='./data/tiny-imagenet-200/val')
    a = dataset.class_to_idx
    reverse_a = {v:k for k,v in a.items()}
    all = list(dataset.class_to_idx.keys())
    f = open('./dataloaders/imagenet_id_to_label.txt', 'r')
    imagenet_id_idx_semantic = f.readlines()
    all_labels = []
    for id in all:
        for line in imagenet_id_idx_semantic:
            if id == line[:-1].split(' ')[0]:
                semantic_label = line[:-1].split(' ')[2]
                all_labels.append(semantic_label)
                break

    return all_labels


def tinyimage_semantic_spit_generator():
    tinyimage_splits = splits_2020["tiny_imagenet"]
    dataset = ImageFolder(root='./data/tiny-imagenet-200/val')
    a = dataset.class_to_idx
    reverse_a = {v:k for k,v in a.items()}
    semantic_splits = [[],[],[],[],[]]
    for i, split in enumerate(tinyimage_splits):
       wnid_split = []
       for idx in split:
           wnid_split.append(reverse_a[idx])
       all = list(dataset.class_to_idx.keys())
       seen = wnid_split
       unseen = list(set(all)-set(seen))
       seen.extend(unseen)
       f = open('./dataloaders/imagenet_id_to_label.txt', 'r')
       imagenet_id_idx_semantic = f.readlines()

       for id in seen:
           for line in imagenet_id_idx_semantic:
               if id == line[:-1].split(' ')[0]:
                   semantic_label = line[:-1].split(' ')[2]
                   semantic_splits[i].append(semantic_label)
                   break
    semantic_splits_detail = {}
    imagenet_id_idx_semantic_detail = json.load(open('dataloaders/tinyimagenet_labels.json'))
    for line in imagenet_id_idx_semantic:
        id = line[:-1].split(' ')[0]
        if id in seen:
            semantic_label = line[:-1].split(' ')[2]
            semantic_splits_detail[semantic_label] = imagenet_id_idx_semantic_detail[id]

    return semantic_splits, semantic_splits_detail


class tinyimage_isolated_class(Dataset):
    def __init__(self, label, mappings, train=False):
        assert label, 'a semantic label should be specified'
        super(tinyimage_isolated_class, self).__init__()
        if train:
            path = './data/tiny-imagenet-200/train/'
        else:
            path = './data/tiny-imagenet-200/val/'
        #path = '/Users/Sepid/data/tiny-imagenet-200/val/'
        self.image_paths = glob.glob(os.path.join(path, mappings[label], '*.JPEG'))
        self.transform = Compose([
            Resize(224, interpolation=Image.BICUBIC),
            CenterCrop(224),
            ToTensor(),
            Normalize((0.4913, 0.4821, 0.4465), (0.2470, 0.2434, 0.2615))
        ])

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, index):
        x = Image.open(self.image_paths[index]).convert('RGB')
        if self.transform:
            x = self.transform(x)
        return x


def tinyimage_single_isolated_class_loader(train=False):
    semantic_splits, semantic_splits_detail = tinyimage_semantic_spit_generator()
    f = open('./dataloaders/tinyimagenet_labels_to_ids.txt', 'r')
    #f = open('../tinyimagenet_ids_to_label.txt', 'r')
    tinyimg_label2folder = f.readlines()
    mappings_dict = {}
    for line in tinyimg_label2folder:
        label, class_id = line[:-1].split(' ')[0], line[:-1].split(' ')[1]
        mappings_dict[label] = class_id

    loaders_dict = {}
    if not train:
        for semantic_label in mappings_dict.keys():
            dataset = tinyimage_isolated_class(semantic_label, mappings_dict)
            loader = DataLoader(dataset=dataset, batch_size=1, num_workers=4)
            loaders_dict[semantic_label] = loader
        return semantic_splits, semantic_splits_detail, loaders_dict
    else:
        all_seen_labels = []
        for split in semantic_splits:
            all_seen_labels += split[:20]
        all_seen_labels = list(set(all_seen_labels))
        for semantic_label in all_seen_labels:
            dataset = tinyimage_isolated_class(semantic_label, mappings_dict)
            loader = DataLoader(dataset=dataset, batch_size=1, num_workers=4)
            loaders_dict[semantic_label] = loader
        return all_seen_labels, loaders_dict


dataloaders/ZO_Clip_loaders.py
Harryqu123/LMC

'''
splits from ARPL (https://github.com/iCGY96/ARPL/blob/master/split.py)
'''
splits_2020 = {
    'cifar10': [
        [0, 6, 4, 9, 1, 7],
        [7, 6, 4, 9, 0, 1],
        [1, 5, 7, 3, 9, 4],
        [8, 6, 1, 9, 0, 7],
        [2, 4, 1, 7, 9, 6]
    ],
    'cifar100': [
        [4, 7, 9, 1],
        [6, 7, 1, 9],
        [9, 6, 1, 7],
        [6, 4, 9, 1],
        [1, 0, 9, 8]
    ],
    'cifar100-10': [
        [30, 25, 1, 9, 8, 0, 46, 52, 49, 71],
        [41, 9, 49, 40, 73, 60, 48, 30, 95, 71],
        [8, 9, 49, 40, 73, 60, 48, 95, 30, 71],
        [95, 60, 30, 73, 46, 49, 68, 99, 8, 71],
        [33, 2, 3, 97, 46, 21, 64, 63, 88, 43]
    ],
    'cifar100-50': [
        [27, 94, 29, 77, 88, 26, 69, 48, 75, 5, 59, 93, 39, 57, 45, 40, 78, 20, 98, 47, 66, 70, 91, 76, 41, 83, 99, 32, 53, 72, 2, 95, 21, 73, 84, 68, 35, 11, 55, 60, 30, 25, 1, 9, 8, 0, 46, 52, 49, 71],
        [65, 97, 86, 24, 45, 67, 2, 3, 91, 98, 79, 29, 62, 82, 33, 76, 0, 35, 5, 16, 54, 11, 99, 52, 85, 1, 25, 66, 28, 84, 23, 56, 75, 46, 21, 72, 55, 68, 8, 69, 41, 9, 49, 40, 73, 60, 48, 30, 95, 71],
        [20, 83, 65, 97, 94, 2, 93, 16, 67, 29, 62, 33, 24, 98, 5, 86, 35, 54, 0, 91, 52, 66, 85, 84, 56, 11, 1, 76, 25, 55, 21, 99, 72, 41, 23, 75, 28, 68, 69, 46, 8, 9, 49, 40, 73, 60, 48, 95, 30, 71],
        [92, 82, 77, 64, 5, 33, 62, 56, 70, 0, 20, 28, 67, 14, 84, 53, 91, 29, 85, 2, 52, 83, 75, 35, 11, 21, 72, 98, 55, 1, 41, 76, 25, 66, 69, 9, 48, 54, 40, 23, 95, 60, 30, 73, 46, 49, 68, 99, 8, 71],
        [47, 6, 19, 0, 62, 93, 59, 65, 54, 70, 34, 55, 23, 38, 72, 76, 53, 31, 78, 96, 77, 27, 92, 18, 82, 50, 98, 32, 1, 75, 83, 4, 51, 35, 80, 11, 74, 66, 36, 42, 33, 2, 3, 97, 46, 21, 64, 63, 88, 43]
    ],
    'tiny_imagenet': [
        [108, 147, 17, 58, 193, 123, 72, 144, 75, 167, 134, 14, 81, 171, 44, 197, 152, 66, 1, 133],
        [198, 161, 91, 59, 57, 134, 61, 184, 90, 35, 29, 23, 199, 38, 133, 19, 186, 18, 85, 67],
        [177, 0, 119, 26, 78, 80, 191, 46, 134, 92, 31, 152, 27, 60, 114, 50, 51, 133, 162, 93],
        [98, 36, 158, 177, 189, 157, 170, 191, 82, 196, 138, 166, 43, 13, 152, 11, 75, 174, 193, 190],
        [95, 6, 145, 153, 0, 143, 31, 23, 189, 81, 20, 21, 89, 26, 36, 170, 102, 177, 108, 169]
    ]
}
dataloaders/split.py
Harryqu123/LMC
import argparse
import torch
import os
from dataloaders.ZO_Clip_loaders import tinyimage_single_isolated_class_loader, tiny_single_isolated_class_dino_loader, tinyimage_semantic_spit_generator
from clip.simple_tokenizer import SimpleTokenizer as clip_tokenizer
from tqdm import tqdm
import numpy as np
from sklearn.metrics import roc_auc_score
import json
from datetime import datetime
import sys
from utils_.utils_ import Logger, compute_oscr
from utils_.clip_utils import tokenize_for_clip
from utils_.dino_utils import extract_features
import time
from tqdm import tqdm
import warnings
warnings.filterwarnings("ignore")

parser = argparse.ArgumentParser("tiny imagenet eval")
parser.add_argument("--gpu_devices", default=1, help="gpu device")
parser.add_argument("--k_images", default=10, type=int)
parser.add_argument("--save_dir", default="output/tiny_imagenet")
parser.add_argument("--image_path", default="output/self_debug/keep")



def image_decoder(clip_model, dino_model, stored_features, k_images, device, image_loaders, split, chatgpt_manual_similar_label, detailed_labels=None):
    seen_labels = split[:20]
    if detailed_labels is not None:
        seen_descriptions = [f"This is a photo of a {detailed_labels[label]}" for label in seen_labels]
    else:
        seen_descriptions = [f"This is a photo of a {label}" for label in seen_labels]

    n_seen = sum([len(image_loaders[label]) for label in seen_labels])
    n_unseen = sum([len(image_loaders[label]) for label in split[20:]])
    targets = torch.tensor(n_seen*[0] + n_unseen*[1])
    

    clip_ood_probs_sum = []
    dino_ood_probs_sum = []
    clip_closeset_probs_sum = []
    dino_closeset_probs_sum = []
    closeset_labels_list = []
    for i, semantic_label in tqdm(enumerate(split)):
        if semantic_label in seen_labels:
            close_set = True
        else:
            close_set = False
        loader = image_loaders[semantic_label]

        # Prepare for dino
        total_labels = seen_labels + chatgpt_manual_similar_label
        total_features = []
        for i in total_labels:
            if i in stored_features: 
                feats = stored_features[i]
            else: 
                feats = None
            if feats is not None:
                if feats.shape[0] < k_images:
                    k = feats.shape[0]
                    k_short = k_images - k
                    n = k_short // k
                    p = k_short % k
                    stack_feat = [feats for n_ in range(n + 1)]
                    stack_feat.append(feats[:p, ...])
                    feats = torch.cat(stack_feat, dim=0)
                    assert feats.shape[0] == k_images
                total_features.append(feats)
            else:
                if i in seen_labels: 
                    raise NotImplementedError("no image for class {}".format(i))
        total_features = torch.cat(total_features, dim=0)
        total_features = total_features.t() # (d, k_images * k_class)

        for idx, image in enumerate(loader):
            
            # CLIP Alignment
            all_desc = seen_descriptions + [f"This is a photo of a {label}" for label in chatgpt_manual_similar_label]
            all_desc_ids = tokenize_for_clip(all_desc, cliptokenizer)

            with torch.no_grad():
                image_feature = clip_model.encode_image(image.cuda()).float()
                image_feature /= image_feature.norm(dim=-1, keepdim=True)
                text_features = clip_model.encode_text(all_desc_ids.cuda()).float()
                text_features /= text_features.norm(dim=-1, keepdim=True)
            zeroshot_probs = (100.0 * image_feature @ text_features.T).softmax(dim=-1).squeeze()

            clip_ood_prob_sum = zeroshot_probs[:20].detach().cpu().numpy()
            clip_ood_probs_sum.append(clip_ood_prob_sum)

            # DINO Alignment
            with torch.no_grad():
                image = image.cuda()
                feats = dino_model(image)
                feats = torch.nn.functional.normalize(feats, dim=1, p=2)

            # softmax then take sum (avg) for each class
            zeroshot_probs_dino = (100.0 * feats @ total_features)
            zeroshot_probs_dino_cls = zeroshot_probs_dino.split(k_images, dim=-1)
            zeroshot_probs_dino_cls = torch.tensor([torch.mean(zeroshot_probs_dino_cls[i]) for i in range(len(total_labels))]).softmax(dim=-1).squeeze()
            ood_prob_sum_dino = zeroshot_probs_dino_cls[:len(seen_labels)].detach().cpu().numpy()
            dino_ood_probs_sum.append(ood_prob_sum_dino)

            if close_set:
                # CLIP
                with torch.no_grad():
                    seen_desc_ids = tokenize_for_clip(seen_descriptions, cliptokenizer)
                    seen_text_feature = clip_model.encode_text(seen_desc_ids.cuda()).float()
                    seen_text_feature /= seen_text_feature.norm(dim=-1, keepdim=True)
                clip_closeset_probs = (100.0 * image_feature @ seen_text_feature.T).softmax(dim=-1).squeeze()
                clip_closeset_probs_sum.append(clip_closeset_probs.detach().cpu().numpy())
                closeset_labels_list.append(seen_labels.index(semantic_label))

                # DINO
                closeset_probs_dino = (100.0 * feats @ total_features[:, :len(seen_labels*k_images)])
                closeset_probs_dino_per_cls_ = closeset_probs_dino.split(k_images, dim=1)
                closeset_probs_dino_per_cls = torch.tensor([torch.mean(closeset_probs_dino_per_cls_[i]) for i in range(len(seen_labels))])
                closeset_probs_dino = closeset_probs_dino_per_cls.softmax(dim=-1).squeeze()
                dino_closeset_probs_sum.append(closeset_probs_dino.detach().cpu().numpy())


    prob = 0.6
    ood_probs_sum_ = [a * prob + b * (1 - prob) for (a, b) in zip(clip_ood_probs_sum, dino_ood_probs_sum)]
    ood_probs_sum = [1 - max(ood_probs_sum_[ii]) for ii in range(len(ood_probs_sum_))]
    auc_sum = roc_auc_score(np.array(targets), np.squeeze(ood_probs_sum))

    closeset_probs_sum = [a * prob + b * (1 - prob) for (a, b) in zip(clip_closeset_probs_sum, dino_closeset_probs_sum)]

    closeset_preds_list = []
    for closeset_prob in closeset_probs_sum:
        closeset_pred = np.argmax(closeset_prob, axis=-1)
        closeset_pred_label = seen_labels[closeset_pred]
        closeset_preds_list.append(seen_labels.index(closeset_pred_label))
    closeset_preds_list = np.array(closeset_preds_list)
    closeset_labels_list = np.array(closeset_labels_list)
    oscr = compute_oscr(np.squeeze(ood_probs_sum)[:n_seen], np.squeeze(ood_probs_sum)[n_seen:], closeset_preds_list, closeset_labels_list)
    print('AUROC = {}, OSCR = {}'.format(auc_sum, oscr))


    return auc_sum, oscr

if __name__ == '__main__':
    args = parser.parse_args()
    k_images = args.k_images
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    save_dir = args.save_dir
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    time_str = datetime.strftime(datetime.now(), '%Y-%m-%d-%H:%M:%S')
    sys.stdout = Logger(os.path.join(args.save_dir, 'eval_{}.log'.format(time_str)))
    print('settings:')
    print(args)

    # prepare dino model
    dino_model = torch.hub.load('facebookresearch/dinov2', "dinov2_vitb14")
    dino_model.cuda()

    state_dict = torch.load("pretrained_model/dinov2_vitb14_pretrain.pth", map_location='cpu')
    state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()}
    state_dict = {k.replace("backbone.", ""): v for k, v in state_dict.items()}
    msg = dino_model.load_state_dict(state_dict, strict=False)
    print('Pretrained weights found at pretrained_model/dinov2_vitb14_pretrain.pth and loaded with msg: {}'.format(msg))
    dino_model.eval()

    # prepare clip model
    # initialize tokenizers for clip and bert, these two use different tokenizers
    clip_model = torch.jit.load("pretrained_model/ViT-B-32.pt").to(device).eval()
    cliptokenizer = clip_tokenizer()

    chatgpt_dict = json.load(open('chat_json/tinyimagenet.json'))

    chatgpt_labels = []
    virtual_labels = []
    for i in range(5):
        virtual_labels.append(chatgpt_dict[str(i)])
        chatgpt_labels += chatgpt_dict[str(i)]
    chatgpt_labels = list(set(chatgpt_labels))

    all_seen_labels = []
    semantic_splits, _ = tinyimage_semantic_spit_generator()
    for split in semantic_splits:
        all_seen_labels += split[:20]
    all_seen_labels = list(set(all_seen_labels))

    stored_features_list = []
    for i in range(5):
        labels = virtual_labels[i]
        labels += semantic_splits[i][:20]
        image_root = os.path.join(args.image_path, str(i))
        classes = os.listdir(image_root)
        for l in labels:
            if l not in classes:
                labels.remove(l)
        tiny_dino_loaders, tiny_dino_labels = tiny_single_isolated_class_dino_loader(tiny_dino_labels=labels, root=image_root)
        stored_features = {}
        for idx_lable, semantic_label in enumerate(tiny_dino_labels):
            print("Extracting features {} {}/{}".format(semantic_label, idx_lable, len(tiny_dino_labels)))
            if semantic_label not in tiny_dino_loaders:
                continue
            stored_features[semantic_label] = extract_features(dino_model, tiny_dino_loaders[semantic_label], k_images)
        print("Finish storing features")
        stored_features_list.append(stored_features)


    splits, detailed_labels, tinyimg_loaders = tinyimage_single_isolated_class_loader()
    print('seen splits:')
    for split in splits:
        print(split[:20])
    
    auc_scores = []
    oscr_scores = []

    for index, split in enumerate(splits):
        chatgpt_labels = virtual_labels[index]
        stored_features = stored_features_list[index]
        auc_list_sum_per_split, oscr_list_sum_per_split = image_decoder(clip_model=clip_model,
                                                                                    dino_model=dino_model, 
                                                                                    stored_features=stored_features,
                                                                                    k_images=k_images,
                                                                                    device=device, 
                                                                                    image_loaders=tinyimg_loaders,
                                                                                    split=split, 
                                                                                    chatgpt_manual_similar_label=chatgpt_labels,
                                                                                    detailed_labels=None)

        auc_scores.append(auc_list_sum_per_split)
        oscr_scores.append(oscr_list_sum_per_split)

    prob = 0.6

    print('Average over 5 splits:')
    print(' AUROC: {} +/- {}, {}'.format(np.mean(auc_scores), np.std(auc_scores), auc_scores))
    print(' OSCR: {} +/- {}, {}'.format(np.mean(oscr_scores), np.std(oscr_scores), oscr_scores))

tinyimagenet_eval_msp.py
Harryqu123/LMC
import torch


def tokenize_for_clip(batch_sentences, tokenizer):
    default_length = 77  # CLIP default
    sot_token = tokenizer.encoder['<|startoftext|>']
    eot_token = tokenizer.encoder['<|endoftext|>']
    tokenized_list = []
    for sentence in batch_sentences:
        text_tokens = [sot_token] + tokenizer.encode(sentence) + [eot_token]
        tokenized = torch.zeros((default_length), dtype=torch.long)
        tokenized[:len(text_tokens)] = torch.tensor(text_tokens)
        tokenized_list.append(tokenized)
    tokenized_list = torch.stack(tokenized_list)
    return tokenized_list
utils_/clip_utils.py
Harryqu123/LMC
import torch

def extract_features(model, data_loader, k_images):
    features = []
    for index, items in enumerate(data_loader):
    # for index, (samples, targets) in enumerate(data_loader):
        # print(type(items))
        if isinstance(items, tuple):
            (samples, targets) = items
        elif isinstance(items, list) and len(items) == 2:
            (samples, targets) = items
        elif isinstance(items, list) and len(items) == 3:
            (samples, targets, _) = items
        elif torch.is_tensor(items):
            samples = items
        if index == k_images: break
        with torch.no_grad():
            samples = samples.cuda()
            feats = model(samples)
            features.append(feats)
        
    features = torch.cat(features, dim=0)
    features = torch.nn.functional.normalize(features, dim=1, p=2)
    print("Storing feature: {}".format(features.shape))
    return features
utils_/dino_utils.py
Harryqu123/LMC
import os
import sys
import errno
import numpy as np
from sklearn.metrics import roc_curve, roc_auc_score, accuracy_score, average_precision_score

def compute_oscr(x1, x2, pred, labels):

    """
    :param x1: open set score for each known class sample (B_k,)
    :param x2: open set score for each unknown class sample (B_u,)
    :param pred: predicted class for each known class sample (B_k,)
    :param labels: correct class for each known class sample (B_k,)
    :return: Open Set Classification Rate
    """

    x1, x2 = -x1, -x2

    # x1, x2 = np.max(pred_k, axis=1), np.max(pred_u, axis=1)
    # pred = np.argmax(pred_k, axis=1)

    correct = (pred == labels)
    m_x1 = np.zeros(len(x1))
    m_x1[pred == labels] = 1
    k_target = np.concatenate((m_x1, np.zeros(len(x2))), axis=0)
    u_target = np.concatenate((np.zeros(len(x1)), np.ones(len(x2))), axis=0)
    predict = np.concatenate((x1, x2), axis=0)
    n = len(predict)

    # Cutoffs are of prediction values

    CCR = [0 for x in range(n + 2)]
    FPR = [0 for x in range(n + 2)]

    idx = predict.argsort()

    s_k_target = k_target[idx]
    s_u_target = u_target[idx]

    for k in range(n - 1):
        CC = s_k_target[k + 1:].sum()
        FP = s_u_target[k:].sum()

        # True	Positive Rate
        CCR[k] = float(CC) / float(len(x1))
        # False Positive Rate
        FPR[k] = float(FP) / float(len(x2))

    CCR[n] = 0.0
    FPR[n] = 0.0
    CCR[n + 1] = 1.0
    FPR[n + 1] = 1.0

    # Positions of ROC curve (FPR, TPR)
    ROC = sorted(zip(FPR, CCR), reverse=True)

    OSCR = 0

    # Compute AUROC Using Trapezoidal Rule
    for j in range(n + 1):
        h = ROC[j][0] - ROC[j + 1][0]
        w = (ROC[j][1] + ROC[j + 1][1]) / 2.0

        OSCR = OSCR + h * w


    return OSCR

def mkdir_if_missing(directory):
    if not os.path.exists(directory):
        try:
            os.makedirs(directory)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

class Logger(object):
    """
    Write console output to external text file.
    Code imported from https://github.com/Cysu/open-reid/blob/master/reid/utils/logging.py.
    """
    def __init__(self, fpath=None):
        self.console = sys.stdout
        self.file = None
        if fpath is not None:
            mkdir_if_missing(os.path.dirname(fpath))
            self.file = open(fpath, 'w')

    def __del__(self):
        self.close()

    def __enter__(self):
        pass

    def __exit__(self, *args):
        self.close()

    def write(self, msg):
        self.console.write(msg)
        if self.file is not None:
            self.file.write(msg)

    def flush(self):
        self.console.flush()
        if self.file is not None:
            self.file.flush()
            os.fsync(self.file.fileno())

    def close(self):
        self.console.close()
        if self.file is not None:
            self.file.close()
utils_/utils_.py
Harryqu123/LMC