app.py

import datetime
import gradio as gr
from langdetect import detect, DetectorFactory, detect_langs
from transformers import pipeline

models = {'en': 'Narsil/deberta-large-mnli-zero-cls', # English
          'de': 'Sahajtomar/German_Zeroshot', # German
          'es': 'Recognai/zeroshot_selectra_medium', # Spanish
          'it': 'joeddav/xlm-roberta-large-xnli', # Italian
          'ru': 'DeepPavlov/xlm-roberta-large-en-ru-mnli', # Russian
          'tr': 'vicgalle/xlm-roberta-large-xnli-anli', # Turkish
          'no': 'NbAiLab/nb-bert-base-mnli'} # Norsk

hypothesis_templates = {'en': 'This example is {}.', # English
                        'de': 'Dieses beispiel ist {}.', # German
                        'es': 'Este ejemplo es {}.', # Spanish
                        'it': 'Questo esempio è {}.', # Italian
                        'ru': 'Этот пример {}.', # Russian
                        'tr': 'Bu örnek {}.', # Turkish
                        'no': 'Dette eksempelet er {}.'} # Norsk

def detect_lang(sequence, labels):
    DetectorFactory.seed = 0
    seq_lang = 'en'

    try:
        seq_lang = detect(sequence)
        lbl_lang = detect(labels)
    except:
        print("Language detection failed!",
              "Date:{}, Sequence:{}, Labels:{}".format(
                  str(datetime.datetime.now()),
                  labels))

    if seq_lang != lbl_lang:
        print("Different languages detected for sequence and labels!",
              "Date:{}, Sequence:{}, Labels:{}, Sequence Language:{}, Label Language:{}".format(
                  str(datetime.datetime.now()),
                  sequence,
                  labels,
                  seq_lang,
                  lbl_lang))

    if seq_lang in models:
        print("Sequence Language detected.",
              "Date:{}, Sequence:{}, Sequence Language:{}".format(
                  str(datetime.datetime.now()),
                  sequence,
                  seq_lang))
    else:
        print("Language not supported. Defaulting to English!",
              "Date:{}, Sequence:{}, Sequence Language:{}".format(
                  str(datetime.datetime.now()),
                  sequence,
                  seq_lang))
        seq_lang = 'en'

    return seq_lang


def sequence_to_classify(sequence, labels):
    label_clean = str(labels).split(",")

    lang = detect_lang(sequence, labels)
    classifier = pipeline("zero-shot-classification",
                          hypothesis_template=hypothesis_templates[lang],
                          model=models[lang])
    response = classifier(sequence, label_clean, multi_label=True)

    predicted_labels = response['labels']
    predicted_scores = response['scores']
    clean_output = {idx: float(predicted_scores.pop(0)) for idx in predicted_labels}
    print("Date:{}, Sequence:{}, Labels: {}".format(
        str(datetime.datetime.now()),
        sequence,
        predicted_labels))

    return clean_output

example_text1 = "Coronavirus disease (COVID-19) is an infectious disease caused by the SARS-CoV-2 virus. Most \
people who fall sick with COVID-19 will experience mild to moderate symptoms and recover without special treatment. \
However, some will become seriously ill and require medical attention."
example_labels1 = "business,health related,politics,climate change"

example_text2 = "Elephants are"
example_labels2 = "big,small,strong,fast,carnivorous"

example_text3 = "Elephants"
example_labels3 = "are big,can be very small,generally not strong enough,are faster than you think"

example_text4 = "Dogs are man's best friend"
example_labels4 = "positive,negative,neutral"

example_text5 = "Amar sonar bangla ami tomay bhalobasi"
example_labels5 = "bhalo,kharap"

example_text6 = "Letzte Woche gab es einen Selbstmord in einer nahe gelegenen kolonie"
example_labels6 = "verbrechen,tragödie,stehlen"

example_text7 = "El autor se perfila, a los 50 años de su muerte, como uno de los grandes de su siglo"
example_labels7 = "cultura,sociedad,economia,salud,deportes"

example_text8 = "Россия в среду заявила, что военные учения в аннексированном Москвой Крыму закончились \
и что солдаты возвращаются в свои гарнизоны, на следующий день после того, как она объявила о первом выводе \
войск от границ Украины."
example_labels8 = "новости,комедия"

example_text9 = "I quattro registi - Federico Fellini, Pier Paolo Pasolini, Bernardo Bertolucci e Vittorio De Sica - \
hanno utilizzato stili di ripresa diversi, ma hanno fortemente influenzato le giovani generazioni di registi."
example_labels9 = "cinema,politica,cibo"

example_text10 = "Ja, vi elsker dette landet,\
som det stiger frem,\
furet, værbitt over vannet,\
med de tusen hjem.\
Og som fedres kamp har hevet\
det av nød til seir"
example_labels10 = "helse,sport,religion,mat,patriotisme og nasjonalisme"

example_text11 = "Şampiyonlar Ligi’nde 5. hafta oynanan karşılaşmaların ardından sona erdi. Real Madrid, \
Inter ve Sporting oynadıkları mücadeleler sonrasında Son 16 turuna yükselmeyi başardı. \
Gecenin dev mücadelesinde ise Manchester City, PSG’yi yenerek liderliği garantiledi."
example_labels11 = "dünya,ekonomi,kültür,sağlık,siyaset,spor,teknoloji"

iface = gr.Interface(
    title="Multilingual Multi-label Zero-shot Classification",
    description="Currently supported languages are English, German, Spanish, Italian, Russian, Turkish, Norsk.",
    fn=sequence_to_classify,
    inputs=[gr.inputs.Textbox(lines=20,
        label="Please enter the text you would like to classify...",
        placeholder="Text here..."),
        gr.inputs.Textbox(lines=5,
        label="Possible candidate labels (separated by comma)...",
        placeholder="Labels here separated by comma...")],
    outputs=gr.outputs.Label(num_top_classes=5),
    capture_session=True,
    #interpretation="default",
    examples=[
        [example_text1, example_labels1],
        [example_text2, example_labels2],
        [example_text3, example_labels3],
        [example_text4, example_labels4],
        [example_text5, example_labels5],
        [example_text6, example_labels6],
        [example_text7, example_labels7],
        [example_text8, example_labels8],
        [example_text9, example_labels9],
        [example_text10, example_labels10],
        [example_text11, example_labels11]]
    )
iface.launch()