Spaces:
Runtime error
Runtime error
File size: 4,989 Bytes
b8be36c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
import streamlit as st
from evaluate import evaluator
import evaluate
import datasets
import sentencepiece
from huggingface_hub import HfApi, ModelFilter
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from transformers import AutoTokenizer, AutoModelForMaskedLM
from transformers import pipeline
import matplotlib.pyplot as plt
st.title("Metric Compare")
st.markdown("## Choose the dataset you want to use for the comparison:")
api = HfApi()
datasets = api.list_datasets(filter="task_categories:text-classification", sort = "downloads", direction=-1, limit = 20)
dset = st.selectbox(datasets)
st.markdown("## Now select up to 5 models to compare their performance:")
models = api.list_models(filter="trained_dataset:imdb", sort = "downloads", direction=-1, limit = 20)
options = st.multiselect(
'Choose your models',
models)
"""
tokenizer1 = AutoTokenizer.from_pretrained("lvwerra/distilbert-imdb")
model1 = AutoModelForSequenceClassification.from_pretrained("lvwerra/distilbert-imdb")
tokenizer2 = AutoTokenizer.from_pretrained("sahn/distilbert-base-uncased-finetuned-imdb")
model2 = AutoModelForSequenceClassification.from_pretrained("sahn/distilbert-base-uncased-finetuned-imdb")
tokenizer3 = AutoTokenizer.from_pretrained("aychang/roberta-base-imdb")
model3 = AutoModelForSequenceClassification.from_pretrained("aychang/roberta-base-imdb")
tokenizer4 = AutoTokenizer.from_pretrained("Sreevishnu/funnel-transformer-small-imdb")
model4 = AutoModelForSequenceClassification.from_pretrained("Sreevishnu/funnel-transformer-small-imdb")
tokenizer5 = AutoTokenizer.from_pretrained("RANG012/SENATOR")
model5 = AutoModelForSequenceClassification.from_pretrained("RANG012/SENATOR")
accuracy = evaluate.load("accuracy")
f1 = evaluate.load('f1')
data = datasets.load_dataset("imdb", split="test").shuffle().select(range(1000))
eval = evaluator("text-classification")
pipe1 = pipeline("text-classification", model=model1, tokenizer= tokenizer1, device=0)
res_accuracy1 = eval.compute(model_or_pipeline=pipe1, data=data, metric=accuracy,
label_mapping={"NEGATIVE": 0, "POSITIVE": 1},)
res_f11 = eval.compute(model_or_pipeline=pipe1, data=data, metric=f1,
label_mapping={"NEGATIVE": 0, "POSITIVE": 1},)
print({**res_accuracy1, **res_f11})
pipe2 = pipeline("text-classification", model=model2, tokenizer= tokenizer2, device=0)
res_accuracy2 = eval.compute(model_or_pipeline=pipe2, data=data, metric=accuracy,
label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
res_f12 = eval.compute(model_or_pipeline=pipe2, data=data, metric=f1,
label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
print({**res_accuracy2, **res_f12})
pipe3 = pipeline("text-classification", model=model3, tokenizer= tokenizer3, device=0)
res_accuracy3 = eval.compute(model_or_pipeline=pipe3, data=data, metric=accuracy,
label_mapping={"neg": 0, "pos": 1},)
res_f13 = eval.compute(model_or_pipeline=pipe3, data=data, metric=f1,
label_mapping={"neg": 0, "pos": 1},)
print({**res_accuracy3, **res_f13})
pipe4 = pipeline("text-classification", model=model4, tokenizer= tokenizer4, device=0)
res_accuracy4 = eval.compute(model_or_pipeline=pipe4, data=data, metric=accuracy,
label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
res_f14 = eval.compute(model_or_pipeline=pipe4, data=data, metric=f1,
label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
print({**res_accuracy4, **res_f14})
pipe5 = pipeline("text-classification", model=model5, tokenizer= tokenizer5, device=0)
res_accuracy5 = eval.compute(model_or_pipeline=pipe5, data=data, metric=accuracy,
label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
res_f15 = eval.compute(model_or_pipeline=pipe5, data=data, metric=f1,
label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
print({**res_accuracy5, **res_f15})
plt.plot(res_accuracy1['accuracy'], res_f11['f1'], marker='o', markersize=6, color="red")
plt.annotate('distilbert', xy=(res_accuracy1['accuracy']+0.001, res_f11['f1']))
plt.plot(res_accuracy2['accuracy'], res_f12['f1'], marker='o', markersize=6, color="blue")
plt.annotate('distilbert-base-uncased-finetuned', xy=(res_accuracy2['accuracy']+0.001, res_f12['f1']))
plt.plot(res_accuracy3['accuracy'], res_f13['f1'], marker='o', markersize=6, color="green")
plt.annotate('roberta-base', xy=(res_accuracy3['accuracy']-0.009, res_f13['f1']))
plt.plot(res_accuracy4['accuracy'], res_f14['f1'], marker='o', markersize=6, color="purple")
plt.annotate('funnel-transformer-small', xy=(res_accuracy4['accuracy']-0.015, res_f14['f1']))
plt.plot(res_accuracy5['accuracy'], res_f15['f1'], marker='o', markersize=6, color="black")
plt.annotate('SENATOR', xy=(res_accuracy5['accuracy']+0.001, res_f15['f1']))
plt.xlabel('Accuracy')
plt.ylabel('F1 Score')
#plt.xlim([0.9, 1.0])
#plt.ylim([0.9, 1.0])
plt.title('Comparing the Models')
""" |