The implementation of "Mitigating Hallucinations and Off-target Machine Translation with Source-Contrastive and Language-Contrastive Decoding"
import logging
import subprocess
import tempfile
import random
import copy
from pathlib import Path
from scripts.utils_run import FLORES101_CONVERT
from sacrebleu import get_source_file
from datasets import load_dataset
from tqdm import tqdm
import os

class MTTask:

    def __init__(self,
                 src_lang: str,
                 tgt_lang: str,
                 testset: str,
                 ):
        self.src_lang = src_lang
        self.tgt_lang = tgt_lang
        self.language_pair = f"{src_lang}-{tgt_lang}"
        self.testset = testset
        base_out_dir = Path(__file__).parent / "out"
        print(base_out_dir)
        assert base_out_dir.exists()
        self.out_dir = base_out_dir / self.testset
        self.out_dir.mkdir(exist_ok=True)

        self.out_dir = self.out_dir / self.language_pair
        self.out_dir.mkdir(exist_ok=True)
        self.load_converter = FLORES101_CONVERT

    def __str__(self):
        return f"{self.testset}-{self.src_lang}-{self.tgt_lang}"

    def evaluate(self, translation_method: callable, type='direct', source_contrastive=1, source_weight=None, language_contrastive=None, language_weight=None) -> Path:

        ## load FLORES dataset
        source_sentences = load_dataset('gsarti/flores_101',self.load_converter[self.src_lang])['devtest']['sentence']

        if type == 'direct':
            translations = translation_method(
            src_lang=self.src_lang,
            tgt_lang=self.tgt_lang,
            source_sentences=source_sentences,
            )
        elif type == 'contrastive':
            multi_source_sentences = [source_sentences]
            src_weights = [1]
            tgt_langs=[self.tgt_lang]
            src_langs=[self.src_lang]
            
            # randomly shuffled input to suppress hallucinations
            for i in range(source_contrastive):
                shuffled_sentences = copy.copy(source_sentences)
                random.shuffle(shuffled_sentences)
                multi_source_sentences.append(shuffled_sentences)
                src_weights.append(source_weight/source_contrastive)
                tgt_langs.append(self.tgt_lang)
                src_langs.append(self.src_lang)

            # input with wrong target language indicator to suppress off-target translation
            if language_contrastive:
                for offtarget in language_contrastive:
                    # ignore contrastive variants that are identical to true translation direction
                    if offtarget == self.tgt_lang:
                        continue
                    # don't create contrastive variant for src language if language is already listed (avoid duplicates)
                    if offtarget == 'src' and self.src_lang in language_contrastive:
                        continue
                    multi_source_sentences.append(source_sentences)
                    src_weights.append(language_weight)
                    if offtarget == 'src':
                        tgt_langs.append(self.src_lang)
                    else:
                        tgt_langs.append(offtarget)
                    src_langs.append(self.src_lang)

            translations = []
            for pair in tqdm(list(zip(*multi_source_sentences))):
                translation = translation_method(
                    src_langs=src_langs,
                    tgt_langs=tgt_langs,
                    src_weights=src_weights,
                    multi_source_sentences=pair,
                    )
                translations.append(translation)
        else:
            raise NotImplementedError

        if type == 'direct':
            file_name = 'direct'
        elif type == 'contrastive':
            file_name = 'contrastive-{0}-{1}'.format(source_contrastive, source_weight)
            if language_contrastive:
                file_name += "-lang-{0}-{1}".format('+'.join(language_contrastive), language_weight)
        else:
            raise NotImplementedError

        with open(str(self.out_dir)+"/"+file_name+".txt", 'w') as f:
            f.write("\n".join(translations))

        if not os.path.isfile(str(self.out_dir)+"/"+"ref.text"):
            target_sentences = load_dataset('gsarti/flores_101', self.load_converter[self.tgt_lang])['devtest'][
                'sentence']
            with open(str(self.out_dir) + "/" + "ref.txt", 'w') as f:
                f.write("\n".join(target_sentences))

        return Path(f.name)

mt_task.py
ZurichNLP/ContraDecode
import logging

import translation_models
from mt_task import MTTask
from translation_models import load_translation_model
from scripts.utils_run import FLORES101_CONVERT, M2M100_TYPES, IGNORE_FOR_FLORES, FAMILIES
from itertools import combinations
import argparse

logging.basicConfig(level=logging.INFO)

def main(args):

    model = load_translation_model(args.model_path, device=0)
    language_pairs = args.language_pairs.split(',')
    language_pairs = [x.split("-") for x in language_pairs]

    if args.oneshot:
        assert isinstance(model, translation_models.llama.LLaMaTranslationModel)
        model.one_shot = True

    tasks = []
    
    for lang_pair in language_pairs:
        tasks.append(MTTask(lang_pair[0],lang_pair[1],'flores'))
        print(f"Task added {lang_pair[0]} - {lang_pair[1]}")

    for task in tasks:
        if args.source_contrastive:
            print(f"Evaluating {task} multi_source")
            out_path = task.evaluate(model.translate_multi_source, 'contrastive', args.source_contrastive, args.source_weight, args.language_contrastive, args.language_weight)
            print(f"Translations saved in {out_path}")
        else:
            print(f"Evaluating {task} direct")
            out_path = task.evaluate(model.translate, 'direct')
            print(f"Translations saved in {out_path}")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--model_path", type=str, default="",
                        help="The HF model path")
    parser.add_argument("--language_pairs", type=str, default="",
                        help="language pairs")
    parser.add_argument("--source_contrastive", nargs='?', const=1, type=int,
                        help="enable contrastive input (randomly shuffled segments from input file). Optional INT defines number of contrastive inputs (default: 1).")
    parser.add_argument("--source_weight", type=float, default=-0.1,
                        help="weight of source-contrastive variant. Default -0.1. If multiple contrastive inputs are used, this defines total weight assigned to them.")
    parser.add_argument("--language_contrastive", type=str, nargs='+', default=None,
                        help="language codes of languages for which to construct contrastive variants. Can be multiple (space-separated); 'src' will be mapped to language code of source language. Example: '--language_contrastive en src' will create two contrastive inputs, one with English, one with the source language as desired output language.")
    parser.add_argument("--language_weight", type=float, default=-0.1,
                        help="weight of contrastive variants with wrong language indicator. Default -0.1. If multiple contrastive inputs are used, this specifies weight assigned to each of them individually.")
    parser.add_argument("--oneshot", action='store_true', default=False,
                        help="For LLaMa: provide one-shot translation example")
    args = parser.parse_args()
    main(args)

scripts/run.py
ZurichNLP/ContraDecode
FLORES101_CONVERT = {
"af": "afr",
"am": "amh",
"ar": "ara",
"hy": "hye",
"as": "asm",
"ast": "ast",
"az": "azj",
"be": "bel",
"bn": "ben",
"bs": "bos",
"bg": "bul",
"my": "mya",
"ca": "cat",
"ceb": "ceb",
"zh": "zho_simpl",
"hr": "hrv",
"cs": "ces",
"da": "dan",
"nl": "nld",
"en": "eng",
"et": "est",
"tl": "tgl",
"fi": "fin",
"fr": "fra",
"ff": "ful",
"gl": "glg",
"lg": "lug",
"ka": "kat",
"de": "deu",
"el": "ell",
"gu": "guj",
"ha": "hau",
"he": "heb",
"hi": "hin",
"hu": "hun",
"is": "isl",
"ig": "ibo",
"id": "ind",
"ga": "gle",
"it": "ita",
"ja": "jpn",
"jv": "jav",
"kea": "kea",
"kam": "kam",
"kn": "kan",
"kk": "kaz",
"km": "khm",
"ko": "kor",
"ky": "kir",
"lo": "lao",
"lv": "lav",
"ln": "lin",
"lt": "lit",
"luo": "luo",
"lb": "ltz",
"mk": "mkd",
"ms": "msa",
"ml": "mal",
"mt": "mlt",
"mi": "mri",
"mr": "mar",
"mn": "mon",
"ne": "npi",
"nso": "nso",
"no": "nob",
"ny": "nya",
"oc": "oci",
"or": "ory",
"om": "orm",
"ps": "pus",
"fa": "fas",
"pl": "pol",
"pt": "por",
"pa": "pan",
"ro": "ron",
"ru": "rus",
"sr": "srp",
"sn": "sna",
"sd": "snd",
"sk": "slk",
"sl": "slv",
"so": "som",
"ku": "ckb",
"es": "spa",
"sw": "swh",
"sv": "swe",
"tg": "tgk",
"ta": "tam",
"te": "tel",
"th": "tha",
"tr": "tur",
"uk": "ukr",
"umb": "umb",
"ur": "urd",
"uz": "uzb",
"vi": "vie",
"cy": "cym",
"wo": "wol",
"xh": "xho",
"yo": "yor",
"zu": "zul"
}

M2M100_TYPES = {
    'af': 1,
    'am': 1,
    'ar': 2,
    'hy': 1,
    'as': 0,
    'ast': 1,
    'az': 1,
    'be': 0,
    'bn': 2,
    'bs': 1,
    'bg': 2,
    'my': 1,
    'ca': 2,
    'ceb': 1,
    'zh': 2,
    'hr': 0,
    'cs': 2,
    'da': 2,
    'nl': 2,
    'en': 3,
    'et': 2,
    'tl': 0,
    'fi': 2,
    'fr': 3,
    'ff': 0,
    'gl': 2,
    'lg': 0,
    'ka': 2,
    'de': 3,
    'el': 2,
    'gu': 1,
    'ha': 1,
    'he': 2,
    'hi': 2,
    'hu': 2,
    'is': 2,
    'ig': 1,
    'id': 2,
    'ga': 1,
    'it': 3,
    'ja': 2,
    'jv': 2,
    'kea': 0,
    'kam': 0,
    'kn': 1,
    'kk': 1,
    'km': 1,
    'ko': 2,
    'ky': 1,
    'lo': 1,
    'lv': 2,
    'ln': 0,
    'lt': 2,
    'luo': 1,
    'lb': 2,
    'mk': 2,
    'ms': 1,
    'ml': 1,
    'mt': 2,
    'mr': 1,
    'mi': 1,
    'mn': 1,
    'ne': 0,
    'nso': 0,
    'no': 2,
    'ny': 1,
    'oc': 0,
    'or': 0,
    'om': 1,
    'ps': 1,
    'fa': 2,
    'pl': 2,
    'pt': 3,
    'pa': 1,
    'ro': 2,
    'ru': 3,
    'sr': 2,
    'sn': 1,
    'sd': 0,
    'sk': 2,
    'sl': 2,
    'so': 1,
    'ku': 1,
    'es': 3,
    'sw': 1,
    'sv': 2,
    'tg': 1,
    'ta': 1,
    'te': 1,
    'th': 2,
    'tr': 2,
    'uk': 2,
    'umb': 1,
    'ur': 1,
    'uz': 0,
    'vi': 2,
    'cy': 1,
    'wo': 0,
    'xh': 1,
    'yo': 1,
    'zu': 1,
}

IGNORE_FOR_FLORES = ['as','kea','kam','ky','luo','mt','mr','ny','om','sn','ku','tg','te','umb']

FAMILIES = {"Germanic":['af','da','nl','de','en','is','lb','no','sv','fy','yi'],'Slavic':['be','bs','bg','hr','cs','mk','pl','ru','sr','sk','sl','uk'],'Turkic':['az','ba','kk','tr','uz']}

scripts/utils_run.py
ZurichNLP/ContraDecode
from unittest import TestCase

from translation_models import load_translation_model


class ContrastiveTargetIDTestCase(TestCase):

    def setUp(self) -> None:
        self.model = load_translation_model("m2m100_418M")

    def test_translate_multi_source_greedy(self):
        translation = self.model.translate_multi_source(
            multi_source_sentences=[
                "I like apples",
                "I like apples",
            ],
            src_langs=2 * ["en"],
            tgt_langs=[
                "de",
                "fr",
            ],
            src_weights=[0.8, 0.2],  # Upweight German target ID
            num_beams=1,
        )
        self.assertEqual(translation, "Ich mag Äpfel")

        translation = self.model.translate_multi_source(
            multi_source_sentences=[
                "I like apples",
                "I like apples",
            ],
            src_langs=2 * ["en"],
            tgt_langs=[
                "de",
                "fr",
            ],
            src_weights=[0.2, 0.8],  # Upweight French target ID
            num_beams=1,
        )
        self.assertEqual(translation, "J’aime les pommes")

    def test_translate_multi_source_beam_search(self):
        translation = self.model.translate_multi_source(
            multi_source_sentences=[
                "I like apples",
                "I like apples",
            ],
            src_langs=2 * ["en"],
            tgt_langs=[
                "de",
                "fr",
            ],
            src_weights=[0.8, 0.2],  # Upweight German target ID
            num_beams=5,
        )
        self.assertEqual(translation, "Ich mag Äpfel")

        translation = self.model.translate_multi_source(
            multi_source_sentences=[
                "I like apples",
                "I like apples",
            ],
            src_langs=2 * ["en"],
            tgt_langs=[
                "de",
                "fr",
            ],
            src_weights=[0.2, 0.8],  # Upweight French target ID
            num_beams=5,
        )
        self.assertEqual(translation, "J’aime les pommes")

tests/test_contrastive_target_id.py
ZurichNLP/ContraDecode
import logging
from unittest import TestCase

from translation_models import load_translation_model
from translation_models.llama import LLaMaTranslationModel


# logging.basicConfig(level=logging.INFO)


class LLaMaTranslationModelTestCase(TestCase):

    def setUp(self) -> None:
        self.llama: LLaMaTranslationModel = load_translation_model("llama-2-7b-chat")
        # self.llama.one_shot = True
        self.assertEqual(self.llama._lang_code_to_name("en"), "English")
        self.assertEqual(self.llama._lang_code_to_name("de"), "German")

    def test_translate(self):
        source_sentences = [
            "Hello, how are you?",
            "An inquiry was established to investigate.",
            "On Monday, scientists from the Stanford University School of Medicine announced the invention of a new "
                "diagnostic tool that can sort cells by type: a tiny printable chip that can be manufactured using "
                "standard inkjet printers for possibly about one U.S. cent each.",
        ]

        for tgt_lang in [
            "de",
            "fr",
            "ru",
        ]:
            translations = self.llama.translate(
                src_lang="en",
                tgt_lang=tgt_lang,
                source_sentences=source_sentences,
                num_beams=1,
            )
            for translation in translations:
                print(translation)

    def test_translate_multi_source(self):
        translation = self.llama.translate_multi_source(
            multi_source_sentences=[
                "I like apples",
                "I like apples",
            ],
            src_langs=2 * ["en"],
            tgt_langs=[
                "de",
                "fr",
            ],
            src_weights=[0.8, 0.2],  # Upweight German target ID
            num_beams=1,
        )
        print(translation)

        translation = self.llama.translate_multi_source(
            multi_source_sentences=[
                "I like apples",
                "I like apples",
            ],
            src_langs=2 * ["en"],
            tgt_langs=[
                "de",
                "fr",
            ],
            src_weights=[0.2, 0.8],  # Upweight French target ID
            num_beams=1,
        )
        print(translation)

tests/test_llama.py
ZurichNLP/ContraDecode
import json
import logging
import os
import warnings

from pathlib import Path
from typing import List, Union, Tuple, Set, Optional

from tqdm import tqdm


class TranslationModel:

    def __str__(self):
        raise NotImplementedError

    def translate(self,
                  tgt_lang: str,
                  source_sentences: Union[str, List[str]],
                  src_lang: str = None,
                  return_score: bool = False,
                  batch_size: int = 8,
                  num_beams: int = 5,
                  **kwargs,
                  ) -> Union[str, List[str], Tuple[str, float], List[Tuple[str, float]]]:
        """
        :param tgt_lang: Language code of the target language
        :param source_sentences: A sentence or list of sentences
        :param src_lang: Language code of the source language (not needed for some multilingual models)
        :param return score: If true, return a tuple where the second element is sequence-level score of the translation
        :param batch_size
        :param kwargs
        :return: A sentence or list of sentences
        """
        if isinstance(source_sentences, str):
            source_sentences_list = [source_sentences]
        elif isinstance(source_sentences, list):
            source_sentences_list = source_sentences
        else:
            raise ValueError

        self._set_tgt_lang(tgt_lang)
        if self.requires_src_lang:
            if src_lang is None:
                warnings.warn(f"NMT model {self} requires the src language. Assuming 'en'; override with `src_lang`")
                src_lang = "en"
            self._set_src_lang(src_lang)
        translations_list = self._translate(source_sentences_list, return_score, batch_size, num_beams=num_beams, **kwargs)
        assert len(translations_list) == len(source_sentences_list)

        if isinstance(source_sentences, str):
            translations = translations_list[0]
        else:
            translations = translations_list
        return translations

    def _translate_multi_source(self,
                                multi_source_sentences: List[str],
                                src_langs: List[str],
                                tgt_langs: List[str],
                                src_weights: Optional[List[float]] = None,
                                **kwargs,
                                ) -> str:
        raise NotImplementedError


    def translate_multi_source(self,
                               multi_source_sentences: List[str],
                               tgt_langs: List[str],
                               src_langs: Optional[List[str]] = None,
                               src_weights: Optional[List[float]] = None,
                               num_beams: int = 5,
                               **kwargs,
                               ) -> str:
        translation = None

        if translation is None:
            self._set_tgt_lang(tgt_langs[0])
            if self.requires_src_lang:
                assert src_langs is not None
            translation = self._translate_multi_source(multi_source_sentences, src_langs, tgt_langs, src_weights=src_weights, num_beams=num_beams, **kwargs)

        return translation


def load_translation_model(name: str, **kwargs) -> TranslationModel:
    """
    Convenience function to load a :class: TranslationModel using a shorthand name of the model
    """
    if name == "m2m100_418M":
        from translation_models.m2m100 import M2M100Model
        translation_model = M2M100Model(model_name_or_path="facebook/m2m100_418M", **kwargs)
    elif name == "m2m100_1.2B":
        from translation_models.m2m100 import M2M100Model
        translation_model = M2M100Model(model_name_or_path="facebook/m2m100_1.2B", **kwargs)
    elif name == "small100":
        from translation_models.small100 import SMaLL100Model
        translation_model = SMaLL100Model(model_name_or_path="alirezamsh/small100", **kwargs)
    elif name == "llama-2-7b-chat":
        from translation_models.llama import LLaMaTranslationModel
        translation_model = LLaMaTranslationModel(model_name_or_path="meta-llama/Llama-2-7b-chat-hf", **kwargs)
    elif name == "llama-2-13b-chat":
        from translation_models.llama import LLaMaTranslationModel
        translation_model = LLaMaTranslationModel(model_name_or_path="meta-llama/Llama-2-13b-chat-hf", **kwargs)
    elif name == "llama-2-70b-chat":
        from translation_models.llama import LLaMaTranslationModel
        translation_model = LLaMaTranslationModel(model_name_or_path="meta-llama/Llama-2-70b-chat-hf", **kwargs)
    else:
        raise NotImplementedError
    return translation_model

translation_models/__init__.py
ZurichNLP/ContraDecode
import logging
from typing import Set, List, Union, Tuple, Optional

import torch
from tqdm import tqdm
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, LogitsProcessorList

from scripts.utils_run import FLORES101_CONVERT
from translation_models import TranslationModel
from translation_models.m2m100 import EnsembleLogitsProcessor
from translation_models.utils_llama import language_names, one_shot_sentences


class LLaMaTranslationModel(TranslationModel):

    # Official templates used during instruction tuning of LLaMA
    TEMPLATE_0 = "{src_sent}\n\nTranslate to {tgt_lang}"
    TEMPLATE_1 = "{src_sent}\n\nCould you please translate this to {tgt_lang}?"
    TEMPLATE_2 = "{src_sent}\n\nTranslate this to {tgt_lang}?"
    TEMPLATE_3 = "Translate to {tgt_lang}:\n\n{src_sent}"
    TEMPLATE_4 = "Translate the following sentence to {tgt_lang}:\n{src_sent}"
    TEMPLATE_5 = "How is \"{src_sent}\" said in {tgt_lang}?"
    TEMPLATE_6 = "Translate \"{src_sent}\" to {tgt_lang}?"

    SYSTEM_PROMPT = """You are a machine translation system that translates sentences from {src_lang} to {tgt_lang}. You just respond with the translation, without any additional comments."""

    def __init__(self,
                 model_name_or_path: str,
                 message_template: str = TEMPLATE_0,
                 one_shot: bool = False,
                 **kwargs,
                 ):
        super().__init__()
        self.model_name_or_path = model_name_or_path
        self.model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map='auto', load_in_4bit=True,
                                                          torch_dtype=torch.bfloat16)
        self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
        self.pipeline = pipeline('text-generation', model=self.model, tokenizer=self.tokenizer)
        self.message_template = message_template
        self.one_shot = one_shot
        self.src_lang = None
        self.tgt_lang = None

    def __str__(self):
        return str(self.model_name_or_path).replace("/", "_")

    @property
    def supported_languages(self) -> Set[str]:
        return {code for code, code3 in FLORES101_CONVERT.items() if code3 in language_names}

    def requires_src_lang(self):
        return True

    def _set_src_lang(self, src_lang: str):
        assert src_lang in self.supported_languages
        self.src_lang = src_lang

    def _set_tgt_lang(self, tgt_lang: str):
        assert tgt_lang in self.supported_languages
        self.tgt_lang = tgt_lang

    def _lang_code_to_name(self, lang_code: str) -> str:
        lang_code3 = FLORES101_CONVERT.get(lang_code, lang_code)
        return language_names[lang_code3]

    @torch.no_grad()
    def _translate(self,
                   source_sentences: List[str],
                   return_score: bool = False,
                   batch_size: int = 1,
                   num_beams: int = 1,
                   **kwargs,
                   ) -> Union[List[str], List[Tuple[str, float]]]:
        if return_score:
            raise NotImplementedError
        if batch_size != 1:
            logging.warning(
                f"Batch size {batch_size} is not supported by LLaMaTranslationModel. Setting batch size to 1.")
            batch_size = 1
        if num_beams != 1:
            logging.warning(f"Beam search is not supported by LLaMaTranslationModel. Setting num_beams to 1.")
            num_beams = 1

        assert self.src_lang is not None
        assert self.tgt_lang is not None
        system_prompt = self.SYSTEM_PROMPT.format(
            src_lang=self._lang_code_to_name(self.src_lang),
            tgt_lang=self._lang_code_to_name(self.tgt_lang),
        )

        if self.one_shot:
            system_prompt += "\n\nExample instruction:\n{instruction}\n\nExample response:\nSure, here's the translation:\n{response}".format(
                instruction=self.message_template.format(
                    src_lang=self._lang_code_to_name(self.src_lang),
                    tgt_lang=self._lang_code_to_name(self.tgt_lang),
                    src_sent=one_shot_sentences[FLORES101_CONVERT.get(self.src_lang, self.src_lang)],
                ),
                response=one_shot_sentences[FLORES101_CONVERT.get(self.tgt_lang, self.tgt_lang)],
            )

        translations = []
        for source_sentence in tqdm(source_sentences):
            prompt_template = PromptTemplate(system_prompt=system_prompt)
            message = self.message_template.format(
                src_lang=self._lang_code_to_name(self.src_lang),
                tgt_lang=self._lang_code_to_name(self.tgt_lang),
                src_sent=source_sentence,
            )
            logging.info(message)
            prompt_template.add_user_message(message)
            prompt = prompt_template.build_prompt()
            prompt += "Sure, here's the translation:\n"
            inputs = self.pipeline.preprocess(prompt)
            output = self.pipeline.forward(
                inputs,
                eos_token_id=self.tokenizer.eos_token_id,
                max_length=1200,  # Max ref length across Flores-101 is 960
                remove_invalid_values=True,
                num_beams=num_beams,
                # Disable sampling
                do_sample=False,
                temperature=1.0,
                top_p=1.0,
            )
            output = self.pipeline.postprocess(output)
            output = output[0]['generated_text']
            logging.info(output)
            prompt_template.add_model_reply(output, includes_history=True)
            response = prompt_template.get_model_replies(strip=True)[0]
            response_lines = response.replace("Sure, here's the translation:", "").strip().split("\n")
            if not response_lines:
                translation = ""
            else:
                translation = response_lines[0].strip()
            translations.append(translation)
        return translations

    def _translate_multi_source(self,
                                multi_source_sentences: List[str],
                                src_langs: List[str],
                                tgt_langs: List[str],
                                src_weights: Optional[List[float]] = None,
                                num_beams: int = 1,
                                **kwargs,
                                ) -> str:
        assert len(multi_source_sentences) == len(src_langs) == len(tgt_langs)
        if src_weights is not None:
            assert len(src_weights) == len(multi_source_sentences)
        if num_beams != 1:
            logging.warning(f"Beam search is not supported by LLaMaTranslationModel. Setting num_beams to 1.")
            num_beams = 1

        prompts = []
        prompt_templates = []
        for src_sent, src_lang, tgt_lang in zip(multi_source_sentences, src_langs, tgt_langs):
            system_prompt = self.SYSTEM_PROMPT.format(
                src_lang=self._lang_code_to_name(src_lang),
                tgt_lang=self._lang_code_to_name(tgt_lang),
            )
            if self.one_shot:
                system_prompt += "\n\nExample instruction:\n{instruction}\n\nExample response:\n{response}".format(
                    instruction=self.message_template.format(
                        src_lang=self._lang_code_to_name(src_lang),
                        tgt_lang=self._lang_code_to_name(tgt_lang),
                        src_sent=one_shot_sentences[FLORES101_CONVERT.get(src_lang, src_lang)],
                    ),
                    response=one_shot_sentences[FLORES101_CONVERT.get(tgt_lang, tgt_lang)],
                )
            prompt_template = PromptTemplate(system_prompt=system_prompt)
            message = self.message_template.format(
                src_lang=self._lang_code_to_name(src_lang),
                tgt_lang=self._lang_code_to_name(tgt_lang),
                src_sent=src_sent,
            )
            prompt_template.add_user_message(message)
            prompt = prompt_template.build_prompt()
            prompt += "Sure, here's the translation:\n"
            prompts.append(prompt)
            prompt_templates.append(prompt_template)

        inputs = [self.pipeline.preprocess(prompt) for prompt in prompts]
        input_ids = [x['input_ids'][0].tolist() for x in inputs]
        attention_mask = [x['attention_mask'][0].tolist() for x in inputs]
        # Add left padding
        max_len = max(len(x) for x in input_ids)
        pad_token_id = self.tokenizer.get_vocab()["▁"]
        input_ids = [[pad_token_id] * (max_len - len(x)) + x for x in input_ids]
        attention_mask = [[0] * (max_len - len(x)) + x for x in attention_mask]
        input_ids = torch.tensor(input_ids).to(self.model.device)
        attention_mask = torch.tensor(attention_mask).to(self.model.device)
        logits_processor = LogitsProcessorList([
            EnsembleLogitsProcessor(num_beams=num_beams, source_weights=src_weights),
        ])
        output = self.model.generate(
            input_ids=input_ids,
            attention_mask=attention_mask,
            num_beams=num_beams,
            eos_token_id=self.tokenizer.eos_token_id,
            max_length=1200,
            logits_processor=logits_processor,
            remove_invalid_values=True,
            # Disable sampling
            do_sample=False,
            temperature=1.0,
            top_p=1.0,
            **kwargs,
        )
        output = output.reshape(1, output.shape[0], *output.shape[1:])
        output = {
            "generated_sequence": output,
            "input_ids": input_ids[0],
            "prompt_text": prompts[0],
        }
        output = self.pipeline._ensure_tensor_on_device(output, device=torch.device("cpu"))
        output = self.pipeline.postprocess(output)
        output = output[0]['generated_text']
        _, output = output.rsplit("[/INST]", maxsplit=1)
        logging.info(output)
        prompt_templates[0].add_model_reply(output, includes_history=False)
        response = prompt_templates[0].get_model_replies(strip=True)[0]
        response_lines = response.replace("Sure, here's the translation:", "").strip().split("\n")
        if not response_lines:
            translation = ""
        else:
            translation = response_lines[0].strip()
        return translation


class PromptTemplate:
    """
    Manages the conversation with a LLaMa chat model.

    Source: https://github.com/samrawal/llama2_chat_templater
    (c) Sam Rawal
    """

    def __init__(self, system_prompt=None):
        self.system_prompt = system_prompt
        self.user_messages = []
        self.model_replies = []

    def add_user_message(self, message: str, return_prompt=True):
        self.user_messages.append(message)
        if return_prompt:
            return self.build_prompt()

    def add_model_reply(self, reply: str, includes_history=True, return_reply=True):
        reply_ = reply.replace(self.build_prompt(), "") if includes_history else reply
        self.model_replies.append(reply_)
        if len(self.user_messages) != len(self.model_replies):
            raise ValueError(
                "Number of user messages does not equal number of system replies."
            )
        if return_reply:
            return reply_

    def get_user_messages(self, strip=True):
        return [x.strip() for x in self.user_messages] if strip else self.user_messages

    def get_model_replies(self, strip=True):
        return [x.strip() for x in self.model_replies] if strip else self.model_replies

    def build_prompt(self):
        if len(self.user_messages) != len(self.model_replies) + 1:
            raise ValueError(
                "Error: Expected len(user_messages) = len(model_replies) + 1. Add a new user message!"
            )

        if self.system_prompt is not None:
            SYS = f"[INST] <<SYS>>\n{self.system_prompt}\n<</SYS>>"
        else:
            SYS = ""

        CONVO = ""
        SYS = "<s>" + SYS
        for i in range(len(self.user_messages) - 1):
            user_message, model_reply = self.user_messages[i], self.model_replies[i]
            conversation_ = f"{user_message} [/INST] {model_reply} </s>"
            if i != 0:
                conversation_ = "[INST] " + conversation_
            CONVO += conversation_

        CONVO += f"[INST] {self.user_messages[-1]} [/INST]"

        return SYS + CONVO

translation_models/llama.py
ZurichNLP/ContraDecode
from typing import List, Union, Tuple, Set, Optional

import torch
import sys
from tqdm import tqdm
from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer, LogitsProcessorList, LogitsProcessor, \
    ForcedBOSTokenLogitsProcessor
from transformers.file_utils import PaddingStrategy

from translation_models import TranslationModel
from translation_models.utils import batch
import torch.nn.functional as F


def zero_out_max(x):
    max_num, max_index = x.max(dim=0)
    x[max_index] = 0
    return x


class EnsembleLogitsProcessor(LogitsProcessor):

    def __init__(self, num_beams: int, source_weights: List[float] = None, preserve_bos_token: bool = False):
        self.num_beams = num_beams
        self.source_weights = source_weights
        self.preserve_bos_token = preserve_bos_token

    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        cur_len = input_ids.shape[-1]
        if self.preserve_bos_token and cur_len <= 1:
            return scores

        scores = F.softmax(scores, dim=-1)

        batch_size = int(input_ids.size(0) / self.num_beams)
        if self.source_weights is not None:
            assert len(self.source_weights) == batch_size
            source_weights = torch.Tensor(self.source_weights).to(scores.device)
        else:
            source_weights = 1/(batch_size-1) * torch.ones((batch_size,), device=scores.device)
        for i in range(self.num_beams):
            beam_indices = self.num_beams * torch.arange(batch_size, device=scores.device, dtype=torch.long) + i
            cands = scores[beam_indices]
            mean_scores = torch.log((source_weights.unsqueeze(-1).expand(-1, scores.size(-1)) * cands).sum(dim=0))
            for j in beam_indices:
                scores[j] = mean_scores

        if torch.isnan(scores).any():
            scores = torch.nan_to_num(scores, nan=float('-inf'))

        return scores


class BatchedForcedBOSTokenLogitsProcessor(ForcedBOSTokenLogitsProcessor):
    r"""
    [`LogitsProcessor`] that enforces the specified token as the first generated token.
    This subclass allows for different forced bos tokens per row in the batch.
    """

    def __init__(self, bos_token_ids: List[int]):
        self.bos_token_ids = bos_token_ids

    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        cur_len = input_ids.shape[-1]
        if cur_len != 1:
            return scores

        batch_size = len(self.bos_token_ids)
        num_beams = int(input_ids.size(0) / batch_size)
        # Disable all tokens by default
        scores.fill_(-float("inf"))
        for row_idx in range(batch_size):
            # Set the bos token for all beams corresponding to this input row
            row_indices = torch.arange(num_beams, device=scores.device) + row_idx * num_beams
            scores[row_indices, self.bos_token_ids[row_idx]] = 0
        return scores


class M2M100Model(TranslationModel):
    """
    Loads one of the models described in: Fan, Angela, et al. "Beyond english-centric multilingual machine
    translation." Journal of Machine Learning Research 22.107 (2021): 1-48.

    Uses the implementation of the Hugging Face Transformers library
    (https://huggingface.co/docs/transformers/model_doc/m2m_100).
    """

    def __init__(self,
                 model_name_or_path: str = "facebook/m2m100_418M",
                 device=None,
                 ):
        self.tokenizer = M2M100Tokenizer.from_pretrained(model_name_or_path)
        self.model_name_or_path = model_name_or_path
        self.model = M2M100ForConditionalGeneration.from_pretrained(model_name_or_path)
        if device is not None:
            self.model = self.model.to(device)
        self.model.config.max_length = max(self.model.config.max_length, self.model.config.max_position_embeddings - 4)

    def __str__(self):
        return self.model_name_or_path

    @property
    def supported_languages(self) -> Set[str]:
        return {'af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'}

    @property
    def requires_src_lang(self) -> bool:
        return True

    def _set_src_lang(self, src_lang: str):
        assert src_lang in self.supported_languages
        self.src_lang = src_lang
        self.tokenizer.src_lang = src_lang

    def _set_tgt_lang(self, tgt_lang: str):
        assert tgt_lang in self.supported_languages
        self.tgt_lang = tgt_lang
        self.tokenizer.tgt_lang = tgt_lang

    @torch.no_grad()
    def _translate(self,
                   source_sentences: List[str],
                   return_score: bool = False,
                   batch_size: int = 8,
                   num_beams: int = 5,
                   **kwargs,
                   ) -> Union[List[str], List[Tuple[str, float]]]:
        padding_strategy = PaddingStrategy.LONGEST if batch_size > 1 else PaddingStrategy.DO_NOT_PAD
        translations = []
        for src_sentences in tqdm(list(batch(source_sentences, batch_size)), disable=len(source_sentences) / batch_size < 10):
            inputs = self.tokenizer._batch_encode_plus(src_sentences, return_tensors="pt",
                                                       padding_strategy=padding_strategy)
            inputs = inputs.to(self.model.device)
            model_output = self.model.generate(
                **inputs,
                forced_bos_token_id=self.tokenizer.get_lang_id(self.tgt_lang),
                num_beams=num_beams,
                return_dict_in_generate=True,
                output_scores=return_score,
                **kwargs,
            )
            batch_translations = self.tokenizer.batch_decode(model_output.sequences, skip_special_tokens=True)
            if return_score:
                # Does not match our score method output for some reason; need to investigate further
                # scores = (2 ** model_output.sequences_scores).tolist()
                scores = [None for _ in batch_translations]
                assert len(batch_translations) == len(scores)
                batch_translations = list(zip(batch_translations, scores))
            translations += batch_translations
        return translations

    def _translate_multi_source(self,
                                multi_source_sentences: List[str],
                                src_langs: List[str],
                                tgt_langs: List[str],
                                src_weights: Optional[List[float]] = None,
                                num_beams: int = 1,
                                **kwargs,
                                ) -> str:
        assert len(multi_source_sentences) == len(src_langs) == len(tgt_langs)
        #src_weights = [1, -0.1]
        if src_weights is not None:
            assert len(src_weights) == len(multi_source_sentences)

        inputs = self.tokenizer._batch_encode_plus(multi_source_sentences, return_tensors="pt",
                                                   padding_strategy=PaddingStrategy.LONGEST)
        # Set individual src language token per row
        for i, src_lang in enumerate(src_langs):
            inputs["input_ids"][i][0] = self.tokenizer.get_lang_id(src_lang)
        inputs = inputs.to(self.model.device)
        logits_processor = LogitsProcessorList([
            BatchedForcedBOSTokenLogitsProcessor([self.tokenizer.get_lang_id(tgt_lang) for tgt_lang in tgt_langs]),
            EnsembleLogitsProcessor(num_beams=num_beams, source_weights=src_weights, preserve_bos_token=True),
        ])
        model_output = self.model.generate(
            **inputs,
            num_beams=num_beams,
            return_dict_in_generate=True,
            logits_processor=logits_processor,
            **kwargs,
        )
        translations = self.tokenizer.batch_decode(model_output.sequences, skip_special_tokens=True)
        return translations[0]

translation_models/m2m100.py
ZurichNLP/ContraDecode
from typing import List, Union, Tuple, Set, Optional
import torch
from tqdm import tqdm
from transformers import LogitsProcessorList, LogitsProcessor
from transformers.file_utils import PaddingStrategy
from transformers.generation_utils import BeamSearchEncoderDecoderOutput
from translation_models import TranslationModel
from translation_models.utils import batch
import torch.nn.functional as F
from translation_models.m2m100 import zero_out_max, EnsembleLogitsProcessor
from transformers import M2M100ForConditionalGeneration
from translation_models.tokenization_small100 import SMALL100Tokenizer


class SMaLL100Model(TranslationModel):
    """
    Uses the implementation of the Hugging Face Transformers library
    (https://huggingface.co/alirezamsh/small100).
    """

    def __init__(self,
                 model_name_or_path: str = "alirezamsh/small100",
                 device=None,
                 ):
        self.tokenizer = SMALL100Tokenizer.from_pretrained(model_name_or_path)
        self.model_name_or_path = model_name_or_path
        self.model = M2M100ForConditionalGeneration.from_pretrained(model_name_or_path)
        if device is not None:
            self.model = self.model.to(device)
        self.model.config.max_length = max(self.model.config.max_length, self.model.config.max_position_embeddings - 4)

    def __str__(self):
        return self.model_name_or_path

    @property
    def supported_languages(self) -> Set[str]:
        return {'af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'}

    @property
    def ranked_languages(self) -> List[str]:
        return ["en", "es", "fr", "de", "pt", "ru", "nl", "sv", "pl", "tr", "id", "zh", "vi", "ar", "el", "cz", "ja", "hu", "fi", "ko", "he", "fa", "lt", "hi"]

    @property
    def requires_src_lang(self) -> bool:
        return False

    def _set_src_lang(self, src_lang: str):
        assert src_lang in self.supported_languages
        self.src_lang = src_lang
        #self.tokenizer.src_lang = src_lang

    def _set_tgt_lang(self, tgt_lang: str):
        assert tgt_lang in self.supported_languages
        self.tgt_lang = tgt_lang
        self.tokenizer.tgt_lang = tgt_lang

    @torch.no_grad()
    def _translate(self,
                   source_sentences: List[str],
                   return_score: bool = False,
                   batch_size: int = 8,
                   num_beams: int = 5,
                   **kwargs,
                   ) -> Union[List[str], List[Tuple[str, float]]]:
        padding_strategy = PaddingStrategy.LONGEST if batch_size > 1 else PaddingStrategy.DO_NOT_PAD
        translations = []
        for src_sentences in tqdm(list(batch(source_sentences, batch_size)), disable=len(source_sentences) / batch_size < 10):
            inputs = self.tokenizer._batch_encode_plus(src_sentences, return_tensors="pt",
                                                       padding_strategy=padding_strategy)
            inputs = inputs.to(self.model.device)
            model_output: BeamSearchEncoderDecoderOutput = self.model.generate(
                **inputs,
                num_beams=num_beams,
                return_dict_in_generate=True,
                output_scores=return_score,
                **kwargs,
            )
            batch_translations = self.tokenizer.batch_decode(model_output.sequences, skip_special_tokens=True)
            if return_score:
                # Does not match our score method output for some reason; need to investigate further
                # scores = (2 ** model_output.sequences_scores).tolist()
                scores = [None for _ in batch_translations]
                assert len(batch_translations) == len(scores)
                batch_translations = list(zip(batch_translations, scores))
            translations += batch_translations
        return translations

    def _translate_multi_source(self,
                                multi_source_sentences: List[str],
                                src_langs: List[str],
                                tgt_langs: List[str],
                                src_weights: Optional[List[float]] = None,
                                num_beams: int = 1,
                                **kwargs,
                                ) -> str:
        assert len(multi_source_sentences) == len(src_langs)
        #src_weights = [0.5,0.25,0.25]

        inputs = self.tokenizer._batch_encode_plus(multi_source_sentences, return_tensors="pt",
                                                   padding_strategy=PaddingStrategy.LONGEST)
        # Set individual src language token per row
        for i, src_lang in enumerate(src_langs):
            inputs["input_ids"][i][0] = self.tokenizer.get_lang_id(tgt_langs[i])
        inputs = inputs.to(self.model.device)
        logits_processor = LogitsProcessorList([EnsembleLogitsProcessor(num_beams=num_beams, source_weights=src_weights)])
        model_output: BeamSearchEncoderDecoderOutput = self.model.generate(
            **inputs,
            num_beams=num_beams,
            return_dict_in_generate=True,
            logits_processor=logits_processor,
            **kwargs,
        )
        translations = self.tokenizer.batch_decode(model_output.sequences, skip_special_tokens=True)
        return translations[0]

translation_models/small100.py
ZurichNLP/ContraDecode
# Copyright (c) 2022 Idiap Research Institute, http://www.idiap.ch/
# Written by Alireza Mohammadshahi <alireza.mohammadshahi@idiap.ch>
# This is a modified version of https://github.com/huggingface/transformers/blob/main/src/transformers/models/m2m_100/tokenization_m2m_100.py 
# which owns by Fariseq Authors and The HuggingFace Inc. team.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenization classes for SMALL100."""
import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from transformers.tokenization_utils import BatchEncoding, PreTrainedTokenizer
from transformers.utils import logging


logger = logging.get_logger(__name__)

SPIECE_UNDERLINE = "▁"

VOCAB_FILES_NAMES = {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "alirezamsh/small100": "https://huggingface.co/alirezamsh/small100/resolve/main/vocab.json",
    },
    "spm_file": {
        "alirezamsh/small100": "https://huggingface.co/alirezamsh/small100/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "alirezamsh/small100": "https://huggingface.co/alirezamsh/small100/resolve/main/tokenizer_config.json",
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "alirezamsh/small100": 1024,
}

# fmt: off
FAIRSEQ_LANGUAGE_CODES = {
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"]
}
# fmt: on


class SMALL100Tokenizer(PreTrainedTokenizer):
    """
    Construct an SMALL100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.
    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        spm_file (`str`):
            Path to [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that
            contains the vocabulary.
        tgt_lang (`str`, *optional*):
            A string representing the target language.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        language_codes (`str`, *optional*):
            What language codes to use. Should be `"m2m100"`.
        sp_model_kwargs (`dict`, *optional*):
            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
            to set:
            - `enable_sampling`: Enable subword regularization.
            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
              - `nbest_size = {0,1}`: No sampling is performed.
              - `nbest_size > 1`: samples from the nbest_size results.
              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
                using forward-filtering-and-backward-sampling algorithm.
            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
              BPE-dropout.
    Examples:
    ```python
    >>> from tokenization_small100 import SMALL100Tokenizer
    >>> tokenizer = SMALL100Tokenizer.from_pretrained("alirezamsh/small100", tgt_lang="ro")
    >>> src_text = " UN Chief Says There Is No Military Solution in Syria"
    >>> tgt_text = "Şeful ONU declară că nu există o soluţie militară în Siria"
    >>> model_inputs = tokenizer(src_text, text_target=tgt_text, return_tensors="pt")
    >>> model(**model_inputs)  # should work
    ```"""

    vocab_files_names = VOCAB_FILES_NAMES
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    model_input_names = ["input_ids", "attention_mask"]

    prefix_tokens: List[int] = []
    suffix_tokens: List[int] = []

    def __init__(
        self,
        vocab_file,
        spm_file,
        tgt_lang=None,
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        pad_token="<pad>",
        unk_token="<unk>",
        language_codes="m2m100",
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        num_madeup_words=8,
        **kwargs,
    ) -> None:
        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

        self.language_codes = language_codes
        fairseq_language_code = FAIRSEQ_LANGUAGE_CODES[language_codes]
        self.lang_code_to_token = {lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code}

        kwargs["additional_special_tokens"] = kwargs.get("additional_special_tokens", [])
        kwargs["additional_special_tokens"] += [
            self.get_lang_token(lang_code)
            for lang_code in fairseq_language_code
            if self.get_lang_token(lang_code) not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            tgt_lang=tgt_lang,
            bos_token=bos_token,
            eos_token=eos_token,
            sep_token=sep_token,
            unk_token=unk_token,
            pad_token=pad_token,
            language_codes=language_codes,
            sp_model_kwargs=self.sp_model_kwargs,
            num_madeup_words=num_madeup_words,
            **kwargs,
        )

        self.vocab_file = vocab_file
        self.encoder = load_json(vocab_file)
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.spm_file = spm_file
        self.sp_model = load_spm(spm_file, self.sp_model_kwargs)

        self.encoder_size = len(self.encoder)

        self.lang_token_to_id = {
            self.get_lang_token(lang_code): self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code)
        }
        self.lang_code_to_id = {lang_code: self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code)}
        self.id_to_lang_token = {v: k for k, v in self.lang_token_to_id.items()}

        self._tgt_lang = tgt_lang if tgt_lang is not None else "en"
        self.cur_lang_id = self.get_lang_id(self._tgt_lang)
        self.set_lang_special_tokens(self._tgt_lang)

        self.num_madeup_words = num_madeup_words

    @property
    def vocab_size(self) -> int:
        return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words

    @property
    def tgt_lang(self) -> str:
        return self._tgt_lang

    @tgt_lang.setter
    def tgt_lang(self, new_tgt_lang: str) -> None:
        self._tgt_lang = new_tgt_lang
        self.set_lang_special_tokens(self._tgt_lang)

    def _tokenize(self, text: str) -> List[str]:
        return self.sp_model.encode(text, out_type=str)

    def _convert_token_to_id(self, token):
        if token in self.lang_token_to_id:
            return self.lang_token_to_id[token]
        return self.encoder.get(token, self.encoder[self.unk_token])

    def _convert_id_to_token(self, index: int) -> str:
        """Converts an index (integer) in a token (str) using the decoder."""
        if index in self.id_to_lang_token:
            return self.id_to_lang_token[index]
        return self.decoder.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens: List[str]) -> str:
        """Converts a sequence of tokens (strings for sub-words) in a single string."""
        return self.sp_model.decode(tokens)

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.
        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.
        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """

        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        prefix_ones = [1] * len(self.prefix_tokens)
        suffix_ones = [1] * len(self.suffix_tokens)
        if token_ids_1 is None:
            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. An MBART sequence has the following format, where `X` represents the sequence:
        - `input_ids` (for encoder) `X [eos, src_lang_code]`
        - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]`
        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
        separator.
        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        if token_ids_1 is None:
            if self.prefix_tokens is None:
                return token_ids_0 + self.suffix_tokens
            else:
                return self.prefix_tokens + token_ids_0 + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        if self.prefix_tokens is None:
            return token_ids_0 + token_ids_1 + self.suffix_tokens
        else:
            return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens

    def get_vocab(self) -> Dict:
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def __getstate__(self) -> Dict:
        state = self.__dict__.copy()
        state["sp_model"] = None
        return state

    def __setstate__(self, d: Dict) -> None:
        self.__dict__ = d

        # for backward compatibility
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = load_spm(self.spm_file, self.sp_model_kwargs)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        save_dir = Path(save_directory)
        if not save_dir.is_dir():
            raise OSError(f"{save_directory} should be a directory")
        vocab_save_path = save_dir / (
            (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
        )
        spm_save_path = save_dir / (
            (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
        )

        save_json(self.encoder, vocab_save_path)

        if os.path.abspath(self.spm_file) != os.path.abspath(spm_save_path) and os.path.isfile(self.spm_file):
            copyfile(self.spm_file, spm_save_path)
        elif not os.path.isfile(self.spm_file):
            with open(spm_save_path, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (str(vocab_save_path), str(spm_save_path))

    def prepare_seq2seq_batch(
        self,
        src_texts: List[str],
        tgt_texts: Optional[List[str]] = None,
        tgt_lang: str = "ro",
        **kwargs,
    ) -> BatchEncoding:
        self.tgt_lang = tgt_lang
        self.set_lang_special_tokens(self.tgt_lang)
        return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)

    def _build_translation_inputs(self, raw_inputs, tgt_lang: Optional[str], **extra_kwargs):
        """Used by translation pipeline, to prepare inputs for the generate function"""
        if tgt_lang is None:
            raise ValueError("Translation requires a `tgt_lang` for this model")
        self.tgt_lang = tgt_lang
        inputs = self(raw_inputs, add_special_tokens=True, **extra_kwargs)
        return inputs

    def _switch_to_input_mode(self):
        self.set_lang_special_tokens(self.tgt_lang)

    def _switch_to_target_mode(self):
        self.prefix_tokens = None
        self.suffix_tokens = [self.eos_token_id]        

    def set_lang_special_tokens(self, src_lang: str) -> None:
        """Reset the special tokens to the tgt lang setting. No prefix and suffix=[eos, tgt_lang_code]."""
        lang_token = self.get_lang_token(src_lang)
        self.cur_lang_id = self.lang_token_to_id[lang_token]
        self.prefix_tokens = [self.cur_lang_id]
        self.suffix_tokens = [self.eos_token_id]

    def get_lang_token(self, lang: str) -> str:
        return self.lang_code_to_token[lang]

    def get_lang_id(self, lang: str) -> int:
        lang_token = self.get_lang_token(lang)
        return self.lang_token_to_id[lang_token]


def load_spm(path: str, sp_model_kwargs: Dict[str, Any]) -> sentencepiece.SentencePieceProcessor:
    spm = sentencepiece.SentencePieceProcessor(**sp_model_kwargs)
    spm.Load(str(path))
    return spm


def load_json(path: str) -> Union[Dict, List]:
    with open(path, "r") as f:
        return json.load(f)


def save_json(data, path: str) -> None:
    with open(path, "w") as f:
        json.dump(data, f, indent=2)

translation_models/tokenization_small100.py
ZurichNLP/ContraDecode
from typing import List


def batch(input: List, batch_size: int):
    l = len(input)
    for ndx in range(0, l, batch_size):
        yield input[ndx:min(ndx + batch_size, l)]

translation_models/utils.py
ZurichNLP/ContraDecode
language_names = {'eng': 'English', 'afr': 'Afrikaans', 'amh': 'Amharic', 'ara': 'Arabic', 'hye': 'Armenian',
    'asm': 'Assamese', 'ast': 'Asturian', 'azj': 'Azerbaijani', 'bel': 'Belarusian', 'ben': 'Bengali', 'bos': 'Bosnian',
    'bul': 'Bulgarian', 'mya': 'Burmese', 'cat': 'Catalan', 'ceb': 'Cebuano', 'zho_simpl': 'Chinese (Simplified)',
    'zho_trad': 'Chinese (Traditional)', 'hrv': 'Croatian', 'ces': 'Czech', 'dan': 'Danish', 'nld': 'Dutch',
    'est': 'Estonian', 'tgl': 'Filipino (Tagalog)', 'fin': 'Finnish', 'fra': 'French', 'ful': 'Fula', 'glg': 'Galician',
    'lug': 'Ganda', 'kat': 'Georgian', 'deu': 'German', 'ell': 'Greek', 'guj': 'Gujarati', 'hau': 'Hausa',
    'heb': 'Hebrew', 'hin': 'Hindi', 'hun': 'Hungarian', 'isl': 'Icelandic', 'ibo': 'Igbo', 'ind': 'Indonesian',
    'gle': 'Irish', 'ita': 'Italian', 'jpn': 'Japanese', 'jav': 'Javanese', 'kea': 'Kabuverdianu', 'kam': 'Kamba',
    'kan': 'Kannada', 'kaz': 'Kazakh', 'khm': 'Khmer', 'kor': 'Korean', 'kir': 'Kyrgyz', 'lao': 'Lao', 'lav': 'Latvian',
    'lin': 'Lingala', 'lit': 'Lithuanian', 'luo': 'Luo', 'ltz': 'Luxembourgish', 'mkd': 'Macedonian', 'msa': 'Malay',
    'mal': 'Malayalam', 'mlt': 'Maltese', 'mri': 'Māori', 'mar': 'Marathi', 'mon': 'Mongolian', 'npi': 'Nepali',
    'nso': 'Northern Sotho', 'nob': 'Norwegian', 'nya': 'Nyanja', 'oci': 'Occitan', 'ory': 'Oriya', 'orm': 'Oromo',
    'pus': 'Pashto', 'fas': 'Persian', 'pol': 'Polish', 'por': 'Portuguese (Brazil)', 'pan': 'Punjabi',
    'ron': 'Romanian', 'rus': 'Russian', 'srp': 'Serbian', 'sna': 'Shona', 'snd': 'Sindhi', 'slk': 'Slovak',
    'slv': 'Slovenian', 'som': 'Somali', 'ckb': 'Sorani Kurdish', 'spa': 'Spanish (Latin America)', 'swh': 'Swahili',
    'swe': 'Swedish', 'tgk': 'Tajik', 'tam': 'Tamil', 'tel': 'Telugu', 'tha': 'Thai', 'tur': 'Turkish',
    'ukr': 'Ukrainian', 'umb': 'Umbundu', 'urd': 'Urdu', 'uzb': 'Uzbek', 'vie': 'Vietnamese', 'cym': 'Welsh',
    'wol': 'Wolof', 'xho': 'Xhosa', 'yor': 'Yoruba', 'zul': 'Zulu', }

# First sentence from "dev" split
one_shot_sentences = {
    'afr': 'Op Maandag het wetenskaplikes vanaf die Stanford Universiteit Medieseskool aangekondig dat die uitvinding van ‘n diagnostiese hulpmiddel wat selle kan rangskik volgens tipe: ‘n klein drukbare skyfie wat vervaardig kan word deur om standard inkjetprinters te gebruik vir ‘n moontlike een VS sent elk.',
    'amh': 'ሰኞ እለት፣ በስታንፎርድ ዩኒቨርሲቲ የህክምና ትምህርት ቤት ህዋሶችን በአይነት የሚያስቀምጥ አዲስ የምርመራ መሳሪያ እንደተፈጠረ አስታውቋል፡ እያንዳንዱን በአንደ የዩ.ኤስ ሳንቲም የሚሆን መደበኛ የኢንክጄት አታሚዎችን በመጠቀም ሊፈበረክ የሚችል ትንሽ መታተም የሚችል ቺፕ።',
    'ara': 'في يوم الاثنين، أعلن علماء من كلية الطب بجامعة ستانفورد عن اختراع أداة تشخيصية جديدة يمكنها تصنيف الخلايا حسب النوع: شريحة صغيرة قابلة للطباعة يمكن تصنيعها باستخدام طابعات نفاثة للحبر قياسية مقابل حوالي سنت أمريكي واحد لكل منها.',
    'hye': 'Երկուշաբթի օրը Բժշկության Ստենֆորդի համալսարանի դպրոցի գիտնականները հայտարարեց նոր ախտորոշման գործիքի ստեղծման մասին, որը կարող է տեսակավորել բջիջներն ըստ տեսակի՝ փոքրիկ տպման հնարավորությամբ չիպ, որը կարող է արտադրվել՝ օգտագործելով ստանդարտ թանաքային տպիչներ յուրաքանչյուրը հավանաբար մոտ մեկ ԱՄՆ ցենտով։',
    'asm': "সোমবাৰে, ষ্টেনফ'ৰ্ড ইউনিভাৰচিটি স্কুল অৱ মেডিচিনৰ বিজ্ঞানীসকলে  ঘোষণা কৰিছিল যে তেওঁলোকে এটা নতুন ৰোগ চিনাক্তকৰণ সঁজুলি আৱিষ্কাৰ কৰিছিল যিয়ে ধৰণ অনুসৰি কোষসমূহ ক্ৰম কৰে: এটা ক্ষুদ্ৰ মুদ্ৰণযোগ্য চিপ যিটো প্ৰত্যেকটোতে সম্ভৱপৰ প্ৰায় 1 U.S. চেণ্ট খৰচ হোৱাকৈ ষ্টেণ্ডাৰ্ড ইনজেক্ট প্ৰিণ্টাৰ ব্যৱহাৰ কৰি প্ৰস্তুত কৰা হৈছে।",
    'ast': 'El llunes, científicos de la Escuela Universitaria de Medicina de la Universidad de Stanford, ficieron anuncia de la invención d’una ferramienta de diagnósticu que puede dixebrar les célules acordies col tipu: un chip imprimible minúsculu que se puede manufacturar usando impresores de tinta estándar pol preciu d’aproximadamente un centavu de dólar per unidá.',
    'azj': 'Bazar ertəsi, Stanford Universitetinin Tibb Fakültəsinin professorları, hüceyrələri növlərinə görə kateqoriyalara bölən yeni bir diaqnoz cihazının ixtira edildiyini bildirib. Bu kiçik çap edilə bilən çip, hər biri təxminən 1 ABŞ sentinə başa gələn standart mürəkkəb kartricli printerlər vasitəsilə istehsal edilə bilər.',
    'bel': 'Навукоўцы медыцынскага факультэту Стэнфардскага ўніверсітэту абвясцілі ў панядзелак аб адкрыцці новага дыягнастычнага сродку, які дазваляе сартаваць клеткі па тыпах. Гаворка ідзе пра маленькі друкаваны чып, які можна надрукаваць на звычайным струменным прынтары. Кошт такога чыпу — каля аднаго цэнту ЗША за адзінку.',
    'ben': 'সোমবার, স্ট্যানফোর্ড বিশ্ববিদ্যালয়ের স্কুল অফ মেডিসিনের বিজ্ঞানীরা একটি নতুন রোগ নির্ণয়ের যন্ত্র আবিষ্কারের ঘোষণা দিয়েছেন যেটি কোষগুলি প্রকারভেদে আলাদা করতে পারে: একটি ছোট মুদ্রণযোগ্য চিপ যার সম্ভাব্য মূল্য প্রায় মার্কিন এক সেন্ট মানসম্পন্ন ইঙ্কজেট প্রিন্টার ব্যবহার করে তৈরি করা যেতে পারে।',
    'bos': 'Naučnici sa Medicinskog fakulteta Univerziteta u Stanfordu su u ponedjeljak najavili izum novog dijagnostičkog alata koji može da sortira stanice prema tipu: maleni čip koji se može štampati i proizvoditi korištenjem standardnih tintnih štampača za oko jedan američki cent.',
    'bul': 'Учените от медицинското училище в университета в Станфод обявиха в понеделник изобретяването на нов диагностичен инструмент, който може да сортира клетките по тип: малък печатен чип, който може да бъде произведен с помощта на стандартни мастилено-струйни принтери за вероятно около един американски цент всеки.',
    'mya': 'တနင်္လာနေ့တွင် စတန်းဖို့ဒ်တက္ကသိုလ် ဆေးကျောင်းမှ သိပ္ပံပညာရှင်များသည် ဆဲလ်များကို အမျိုးအစားအလိုက် စီစဉ်နိုင်သော ရောဂါခွဲခြားစစ်ဆေးမှု ကိရိယာအသစ် တီထွင်မှုအကြောင်းကို ကြေညာခဲ့သည်- ၎င်းသည် စံနှုန်းမီ မင်စက်ကလေးများဖြင့် ပုံဖော်သည့် ပရင်တာများကို သုံးကာ ထုတ်လုပ်နိုင်သော အလွန်သေးငယ်သည့် တစ်ခုလျှင် U.S. ဆင့်တစ်ပြားသာသာရှိသော ပရင့်လုပ်၍ရသော အပြားလေးဖြစ်ပါသည်။',
    'cat': "Dilluns, científics de la Facultat de Medicina de la Universitat de Stanford van anunciar la invenció d'una nova eina per fer diagnòstics que pot separar les cèl·lules per tipus: un petit xip que pot ser imprès i fabricat usant impressores d'injecció estàndards per probablement un cèntim americà cadascun.",
    'ceb': 'Kaniadtong Lunes, gipahibalo sa mga siyentista gikan sa Stanford University School of Medicine ang pag-imbento sa usa ka bag-ong himan sa pagdayagnos nga makahimo sa paghan-ay sa mga selyula pinaagi sa lahi: usa ka gamay nga maimprinta nga tipik nga mahimong gamit ang sagad nga mga inkjet printer alang sa tingali mga usa ka sentimo sa Estados Unidos.',
    'zho_simpl': '周一，斯坦福大学医学院的科学家宣布，他们发明了一种可以将细胞按类型分类的新型诊断工具：一种可打印的微型芯片。这种芯片可以使用标准喷墨打印机制造，每片价格可能在一美分左右。',
    'hrv': 'U ponedjeljak su znanstvenici s Medicinskog fakulteta Sveučilišta u Stanfordu najavili izum novog dijagnostičkog alata koji omogućuje sortiranje stanica prema tipu: to je sićušni čip koji je moguće isprintati s pomoću običnog tintnog pisača za cijenu od oko jedan američki cent po komadu.',
    'ces': 'V pondělí vědci z Lékařské fakulty Stanfordovy univerzity oznámili vynález nového diagnostického nástroje, který dokáže třídit buňky podle typu: malý vytisknutelný čip, který lze vyrobit pomocí standardních inkoustových tiskáren za cenu přibližně jednoho amerického centu za kus.',
    'dan': 'Mandag offentliggjorde forskere fra Stanford Universitys School of Medicine opfindelsen af et nyt diagnoseværktøj, som kan sortere celler efter type: en ganske lille printbar chip, som kan fremstilles med almindelige inkjetprintere, muligvis for omkring én US-cent stykket.',
    'nld': 'Op maandag kondigden wetenschappers van de Stanford University School of Medicine aan dat een nieuw diagnostisch hulpmiddel is uitgevonden dat cellen op type kan ordenen: een piepkleine af te drukken chip die voor waarschijnlijk ongeveer één dollarcent per stuk kan worden geproduceerd met behulp van standaard inkjetprinters.',
    'eng': 'On Monday, scientists from the Stanford University School of Medicine announced the invention of a new diagnostic tool that can sort cells by type: a tiny printable chip that can be manufactured using standard inkjet printers for possibly about one U.S. cent each.',
    'est': 'Esmaspäeval teatasid Stanfordi ülikooli meditsiiniteaduskonna teadlased, et leiutasid uue diagnostilise tööriista, millega saab rakke tüübi järgi sortida; see on väike prinditav kiip, mida saab toota standardsete tindiprinteritega ja ühe hind võib olla vaid 1USA sent.',
    'tgl': 'Noong Lunes, inanunsiyo ng mga siyentipiko mula sa Stanford University School of Medicine ang imbensyon ng panibagong kagamitan sa pag-diagnose na makakauri sa mga cell ayon sa uri: isang maliit na chip na maaaring maprint na maaaring magawa gamit ang standard inkjet na mga printer at posibleng nasa isang U.S. sentimo kada isa.',
    'fin': 'Stanfordin yliopiston lääketieteen laitoksen tutkijat ilmoittivat maanantaina uuden diagnostiikkatyökalun keksimisestä: solut tyypin mukaan lajitteleva pienenpieni tulostettava siru, joka voidaan valmistaa normaaleilla mustesuihkutulostimilla mahdollisesti noin yhden Yhdysvaltain sentin kappalehintaan.',
    'fra': "Des scientifiques de l’école de médecine de l’université de Stanford ont annoncé ce lundi la création d'un nouvel outil de diagnostic, qui permettrait de différencier les cellules en fonction de leur type. Il s'agit d'une petit puce imprimable, qui peut être produite au moyen d'une imprimante à jet d'encre standard, pour un coût d'environ un cent de dollar pièce.",
    'ful': 'Nyande Altine, himɓe ekkititta hala tsari duniya je huwata ha Makaranta dokotoro en je Stanfod wi kuje kesum man je heftata iri kujeji ɓandu kuje mari yonki wawan maha ɗum be kujeji noone printa ɗum nanga sisi 1.',
    'glg': 'O luns, científicos da Escola de Medicina da Universidade de Stanford anunciaron a creación dunha nova ferramenta diagnóstica que pode clasificar as células polo seu tipo: un minúsculo chip imprimible que se pode fabricar empregando impresoras de inxección de tinta normais cun custo de probablemente un céntimo de dólar estadounidense cada un.',
    'lug': "Ku balaza, Banasayansi okuva mu setendekero ya Stanford ku somero ly'ebyedagala balangirira okuvumbulwa kwa akuuma akakebera nga kasobola okusengeka obutafaali nga kasinzira kukika kyabwo: Akuuma katono akasobola okufulumizibwa ku lupapula akasobola okukolebwa ne Printa enungi ku sente entono nga emu eya US buli kamu.",
    'kat': 'ორშაბათს, სტენფორდის უნივერსიტეტის სამედიცინო კათედრის მეცნიერებმა გააკეთეს განცხადება ახალი დიაგნოსტიკური ხელსაწყოს გამოგონების შესახებ, რომელსაც უჯრედის ტიპებად დაყოფა შეუძლია: წვრილი, დასაბეჭდი ჩიპი, რომლის წარმოებაც შესაძლებელია ჩვეულებრივი ჭავლური პრინტერის საშუალებით. ერთი ცალის სავარაუდო ღირებულება აშშ ცენტს შეადგენს.',
    'deu': 'Am Montag haben die Wisenschaftler der Stanford University School of Medicine die Erfindung eines neuen Diagnosetools bekanntgegeben, mit dem Zellen nach ihrem Typ sortiert werden können: ein winziger, ausdruckbarer Chip, der für jeweils etwa einen US-Cent mit Standard-Tintenstrahldruckern hergestellt werden kann.',
    'ell': 'Τη Δευτέρα, επιστήμονες από την Ιατρική Σχολή του Πανεπιστημίου του Στάνφορντ ανακοίνωσαν την εφεύρεση ενός νέου εργαλείου διάγνωσης με δυνατότητα ομαδοποίησης των κυττάρων ανά τύπο: ένα μικροσκοπικό εκτυπώσιμο τσιπ που μπορεί να κατασκευαστεί με απλούς εκτυπωτές ψεκασμού μελάνης με κόστος περίπου ένα σεντ του αμερικανικού δολαρίου το καθένα.',
    'guj': 'સોમવારે, સ્ટેનફોર્ડ યુનિવર્સિટી સ્કૂલ ઓફ મેડિસિનના વૈજ્ઞાનિકોએ એક નવા નિદાન સાધનની શોધની જાહેરાત કરી હતી જે કોષોને પ્રકાર દ્વારા ક્રમમાં ગોઠવી શકે છેઃ એક નાનકડી પ્રિન્ટેબલ ચિપ કે જે સંભવતઃ એક અમેરિકન સેન્ટ માટે સ્ટાન્ડર્ડ ઇન્કજેટ પ્રિન્ટર્સનો ઉપયોગ કરીને બનાવી શકાય.',
    'hau': "A ranar Litinin, masana kimiyya daga Kwalejin Kimiyya ta Jami'ar Stanford suka sanar da kirkirar sabon kayan aikin bincike wanda zai iya rarrabe kwayoyin hallita ta nau'i: ƙaramin cip mai buguwa wanda za'a iya kera ta yin amfani da daidaitattun firinta na inkjet don yiwuwar kusan Amurka ɗaya kowannensu.",
    'heb': 'ביום שני, מדענים מבית הספר לרפואה באוניברסיטת סטנפורד הכריזו על המצאת אמצעי אבחון חדש אשר מסוגל למיין תאים לפי סוג: שבב זעיר הניתן להדפסה שאפשר לייצר באמצעות מדפסות הזרקת דיו סטנדרטיות בעלות של בערך סנט אמריקאי אחד לכל פריט.',
    'hin': 'सोमवार को, स्टैनफ़ोर्ड यूनिवर्सिटी स्कूल ऑफ़ मेडिसिन के वैज्ञानिकों ने एक नए डायग्नोस्टिक उपकरण के आविष्कार की घोषणा की जो कोशिकाओं को उनके प्रकार के आधार पर छाँट सकता है: एक छोटी प्रिंट करने योग्य चिप जिसे स्टैण्डर्ड इंकजेट प्रिंटर का उपयोग करके लगभग एक अमेरिकी सेंट के लिए निर्मित किया जा सकता है.',
    'hun': 'Hétfőn a Stanford Egyetem Orvostudományi Kara bejelentette egy új diagnosztikai eszköz feltalálását, amely képes típus szerint rendszerezni a sejteket: ez egy apró nyomtatható csip, amelyet szabványos tintasugaras nyomtatóval lehet előállítani, akár darabját egy amerikai centért.',
    'isl': 'Á mánudag tilkynntu vísindamenn frá læknadeild Stanford-háskóla uppfinningu á nýju greiningartæki sem getur flokkað frumur eftir tegund: örlítill prentanleg flaga sem hægt er að framleiða með venjulegum bleksprautuprentara fyrir mögulega um eitt bandarískt sent stykkið.',
    'ibo': 'Na Monde, ndi oka mmuta sayensi sitere na Mahadum Stanford Ulo-akwukwo nke Ogwu kwuputara nchoputa nke ngwa-oru nyochaputa ohuu nwere it hoputa mkpuru-ahu site udi: otu mpekere ngwa nka nke enwere ike iruputa site na iji inkjeti igwe-prita din mma nke enwere-ike imeputa na otu senti mba US n’otu.',
    'ind': 'Ilmuwan dari Stanford University School of Medicine pada hari Senin mengumumkan penemuan alat diagnostik baru yang bisa mengurutkan sel berdasarkan tipe: cip kecil dapat dicetak yang bisa diproduksi menggunakan printer inkjet standar dengan biaya sekitar satu sen AS per cip.',
    'gle': "Ar an Luan, d’fhógair eolaithe ó Scoil Leighis Ollscoil Stanford go rabhthas tar éis uirlis nua dhiagnóiseach a dhéanamh ar féidir léi cealla a shórtáil de réir cineáil: sliseanna beag inphriontáilte is féidir a mhonarú le scairdphrintéirí ar thart ar ceint amháin S.A.. b'fhéidir.",
    'ita': "Nella giornata di lunedì, alcuni scienziati della Scuola di Medicina dell'Università di Stanford hanno annunciato l'invenzione di un nuovo strumento diagnostico capace di ordinare le cellule in base al tipo: un chip minuscolo che può essere stampato utilizzando stampanti a getto di inchiostro al costo di circa 1 centesimo di dollaro l'uno.",
    'jpn': '月曜日にスタンフォード大学医学部の科学者たちは、細胞を種類別に分類できる新しい診断ツールを発明したと発表しました。それは標準的なインクジェットプリンタで印刷して製造できる小型チップであり、原価は1枚あたり1円ほどす。',
    'jav': 'Ing dina Senin, ilmuwan saka Fakultas Kedokteran Universitas Stanford ngumumake penemuan piranti diagnosa sing isa misah-misahake sel adhedhasar jenis: kepingan cilik sing isa dicetak sing isa diasilake kanthi nggunakake printer inkjet standar kanggo mungkin saben-saben watara saji sen AS.',
    'kea': 'Na sigunda-fera, sientistas di Skóla di Midisina di Universidadi di Stanford anúnsia invenson di un novu faraménta di diagnótiku ki pode klasifika sélulas pa tipu: un xiipi pikinoti ki ta inprimi ki pode ser fabrikadu ku uzu di inprisoras di jatu di tinta padron pa pusivelmenti serka di un séntimu merkanu kada.',
    'kam': 'wakwambiliilya, anasayanzi kuma sukulu wa stanford wa dawa matangaasie kuseuvya kwa muio mweu wa kuthima ula utonya kuvathukanya cell na muvae : kachip kaini katonya uprinitiwa kala katonya usoovwa kutumia printa ya inkjet standard kwa centi imwe ya U.S.',
    'kan': 'ಅಂದಾಜು ತಲಾ ಒಂದು ಯು.ಎಸ್\u200c ಸೆಂಟ್\u200cನಿಂದ ಪ್ರಮಾಣಿತ ಇಂಕ್\u200cಜೆಟ್ ಪ್ರಿಂಟರುಗಳನ್ನು ಬಳಸಿ ಉತ್ಪಾದಿಸಬಹುದಾದ ಸಣ್ಣ ಪ್ರಿಂಟ್ ಮಾಡಬಹುದಾದ ಚಿಪ್\u200c ಆಗಿರುವ ಸೆಲ್\u200cಗಳ ವಿಧವನ್ನು ಆಯೋಜಿಸುವ ಹೊಸ ಪತ್ತೆ ಪರಿಕರದ ಅನ್ವೇಷಣೆಯ ಘೋಷಣೆಯನ್ನು ಸ್ಟಾನ್\u200cಫೋರ್ಡ್\u200c ವಿಶ್ವವಿದ್ಯಾಲಯದ ಔಷಧ ಶಾಲೆಯ ವಿದ್ಯಾರ್ಥಿಗಳು ಸೋಮವಾರ ಮಾಡಿದ್ದಾರೆ.',
    'kaz': 'Дүйсенбі күні Стэнфорд университетінің медицина факультетінің ғалымдары жасушаларды түрі бойынша сұрыптай алатын жаңа диагностикалық құралды ойлап тапқанын жариялады: бұл әрбірін шамамен бір АҚШ центке стандартты сиялы принтерлерді пайдаланып өндіруге болатын өте кішкентай басып шығаруға болатын чип.',
    'khm': 'កាល\u200bពី\u200bថ្ងៃ\u200bច័ន្ទ អ្នកវិទ្យាសាស្ត្រ\u200bមក\u200bពី\u200bសាលា\u200bវេជ្ជសាស្ត្រ\u200bនៃ\u200bសាកល\u200bវិទ្យាល័យ ស្តែន\u200bហ្វ័ឌ (Stanford University School of Medicine) បាន\u200bប្រកាស\u200bពី\u200bការបង្កើត\u200bឧបករណ៍\u200bធ្វើ\u200bរោគ\u200bវិនិច្ឆ័យ\u200bថ្មី\u200bមួយ ដែល\u200bអាច\u200bតម្រៀប\u200bកោសិកា\u200bតាម\u200bប្រភេទ៖ បន្ទះ\u200bឈីប\u200bដែល\u200bអាច\u200bព្រីន\u200bបាន\u200bដ៏\u200bតូច\u200bមួយ ដែល\u200bអាច\u200bផលិត\u200bដោយ\u200bប្រើ\u200bម៉ាស៊ីន\u200bព្រីន\u200bប្រភេទinkjectស្តង់ដារ ដែល\u200bមាន\u200bតម្លៃ\u200bប្រហែល\u200bមួយ\u200bសេន\u200bសហរដ្ឋអាមេរិក។',
    'kor': '스탠포드 의과대학 연구진은 지난 월요일 세포를 유형별로 분류할 수 있는 새로운 진단도구를 개발했다고 밝혔다. 이는 아주 작은 크기의 인쇄가 가능한 칩으로, 일반적인 잉크젯 프린터를 이용해 개 당 미화 약 1센트로 생산이 가능할 것으로 예상된다.',
    'kir': 'Стэнфорд жогорку окуу жайынын Медицина мектебинин кызматкерлери дүйшөмбүдө клеткаларды параметрлери боюнча иреттей турган жаңы диагностикалык ыкманы ойлоп табышканын айтышты: ал болгону бир АКШ центине даярдала турган чачма принтерлер менен басылып чыгуучу чакан чип.',
    'lao': 'ໃນວັນຈັນນັກວິທະຍາສາດຈາກໂຮງຮຽນແພດສາດມະຫາວິທະຍາໄລສະແຕນຝອດໄດ້ເປີດເຜີຍເຖິງການປະດິດເຄື່ອງມືກວດພະຍາດແບບໃໝ່ທີ່ສາມາດຈັດແບ່ງຈຸລັງຕາມປະເພດ: ແຜ່ນຊິບຂະໜາດນ້ອຍໆທີ່ສາມາດຜະລິດໄດ້ໂດໃຊ້ເຄື່ອງພິມໝຶກມາດຕະຖານເຊິ່ງລາຄາອາດຈະປະມານໜຶ່ງເຊັນສະຫະລັດຕໍ່ອັນ.',
    'lav': 'Stenforda Universitātes Medicīnas skolas zinātnieki pirmdien paziņoja, ka izgudrojuši jaunu diagnostikas rīku, ar kuru var šūnas kārtot pēc to veidiem: tas ir sīks drukājams čips, kuru var izgatavot, izmantojot standarta strūklprinteri, un kura iespējamās izmaksas ir aptuveni viens ASV\xa0cents gabalā.',
    'lin': 'Na mokolo ya yambo, bato ya siansi ya universite ya kimonganga ya Stanford balobaki basali esaleli ya sika oyo ekoki kosalisa na kokabola baselile na kolanda lolenge na yango: mwa eloko moko oyo bakoki kosala na lisalisi ya ba imprimantes à jet ya encre po ezala na talo ya cent moko ya Etats-Unis.',
    'lit': 'Pirmadienį Stanfordo universiteto medicinos mokyklos mokslininkai paskelbė apie naujo diagnostikos įrankio, galinčio rūšiuoti ląsteles pagal tipą, išradimą: mažas spausdintinis lustas, kurį galima gaminti naudojant standartinius rašalinius spausdintuvus. Vieno kaina maždaug vienas JAV dolerio centas.',
    'luo': "Chieng' Wuoktich, josayans mawuok e Mbalariany mar Stanford e Skul mar Thieth nolando ni negifwenyo gimanyien mitiyogo e nono tuoche ma nyalo pogo ng'injo mag del kaluwore kod kitgi: en chip moro matin ma inyalo go chapa gi printa ma bende inyalo losi kitiyo kod printa mapile mag inkjet kwom manyalo romo otonglo achiel mar Amerka e moro ka moro.",
    'ltz': "E Méindeg hu Wëssenschaftler vun der Stanford University School of Medicine d'Erfindung vun engem neien Diagnosgeschier ugekënnegt, dat Zellen no Typ sortéiere kann: e klengen dréckbaren Chip, dee mat Hëllef vun engem Standardtëntestraldrécker fir méiglecherweis ongeféier een US-Cent d'Stéck hiergestallt ka ginn.",
    'mkd': 'Во понеделникот научниците од медицинскиот факултет на Универзитетот Стенфорд објавија дека измислиле нова алатка за дијагностицирање со којашто се подредуваат клетките според типови: се работи за мал чип за печатење кој може да се произведе со помош на стандардни инк-џет печатачи, и тоа по цена од околу еден американски цент.',
    'msa': 'Pada hari Isnin, Saintis daripada Sekolah Perubatan Universiti Stamford mengumumkan penemuan alat diagnostik baru yang boleh mengasingkan sel-sel mengikut jenis: cip kecil yang boleh dicetak yang boleh dihasilakn menggunakan pencetak standard inkjet untuk kira-kira satu sen A.S setiap satu.',
    'mal': 'തിങ്കളാഴ്ച്ച, സ്റ്റാൻഫോർഡ് യൂണിവേഴ്\u200cസിറ്റി സ്\u200cകൂൾ ഓഫ് മെഡിസിനിലെ ശാസ്ത്രജ്ഞന്മാർ കോശങ്ങളെ അവയുടെ ഇനം അനുസരിച്ച് തരംതിരിക്കാൻ കഴിയുന്ന ഒരു പുതിയ രോഗനിർണയ ഉപകരണം കണ്ടുപിടിച്ചതായി പ്രഖ്യാപിച്ചു: സ്റ്റാൻഡേർഡ് ഇങ്ക്\u200cജെറ്റ് പ്രിന്റ്ററുകൾ ഉപയോഗിച്ച് നിർമ്മിക്കാൻ സാധിക്കുന്ന ഏകദേശം ഒരു യു.എസ് സെന്റ് ഓരോന്നിനും വേണ്ടിവരുന്ന പ്രിന്റ് ചെയ്യാൻ കഴിയുന്ന ഒരു ചെറിയ ചിപ്പ്.',
    'mlt': 'Nhar it-Tnejn, xjentisti mill-Iskola tal-Mediċina tal-Università ta’ Stanford ħabbru l-invenzjoni ta’ għodda dijanjostika ġdida li tista’ tirranġa ċ-ċelloli skont it-tip: ċippa ċkejkna li tista’ tiġi stampata li tista’ tiġi manifatturata bl-użu ta’ inkjet printers standard għal possibilment madwar ċenteżmu tal-Istati Uniti kull waħda.',
    'mri': 'I te Mane, i kī ake ngā kaipūtaiao nō Stanford University School of Medicine mō te hanganga o tētahi taputapu whakatau e āhei ai te wewete i ngā pūtau ki ana momo: mō te 1 hēneti U.S pea, he rehu-mihini tā iti nei e taea ana te hanga mā ngā mihini tā inkjet noa.',
    'mar': 'स्टॅनफोर्ड युनिव्हर्सिटी स्कूल ऑफ मेडिसीनमधील शास्त्रज्ञांनी पेशींचे प्रकारानुसार विभाजन करणाऱ्या निदान साधनाचा शोध लावल्याचे सोमवारी जाहीर केलेः सामान्य इंकजेट प्रिंटरचा वापर करून एका लहान प्रिंट करता येणाऱ्या चीपचे उत्पादन अंदाजे एक U.S. प्रति सेंट खर्चून केले जाऊ शकते.',
    'mon': 'Даваа гарагт Стэнфордын Их Сургуулийн Анагаахын сургуулийн эрдэмтэд эсийг төрлөөр нь эрэмбэлж чаддаг шинэ оношилгооны багаж бүтээснээ зарлалаа: уг багаж нь тухай бүрийг нь нэг цент ам.долларын өртгөөр стандарт бэхэн хэвлэгч ашиглан үйлдвэрлэж болдог жижигхэн хэвлэж болдог чип юм.',
    'npi': 'सोमबारका दिन, स्ट्यानफोर्ड युनिभर्सिटी स्कुल अफ मेडिसिनका वैज्ञानिकहरूले एक नयाँ डायग्नोस्टिक उपकरणको आविष्कारको घोषणा गरे जसले कोषहरूलाई प्रकारका आधारमा क्रमबद्ध गर्न सक्दछः एउटा सानो प्रिन्ट गर्न सकिने चिप जुन मानक ईंकजेट प्रिन्टरहरू प्रयोग गरेर सम्भवतः लगभग एक अमेरिकी सेन्टको लागतमा निर्माण गर्न सकिन्छ।',
    'nso': 'Ka Mošupulogo, boramahlale ba go tšwa Sekolong sa Yunibesithi ya Stanford sa Medicine ba tsebišitše ka go dirwa ga sedirišwa se seswa sa tekolo seo se ka beakanyago disele ka mehuta: chip yeo e gatišegago ye nnyane yeo e ka dirwago go šomišwa di printer tša inkjet tša sente ye tee go U.S.',
    'nob': 'Forskere fra Stanford University School of Medicine gjorde kunngjørelsen på mandag om oppfinnelsen av et nytt diagnoseverktøy som sorterer celler etter type: en ørliten utskrivbar chip kan produseres ved hjelp av standard inkjettskrivere for muligens ca. én amerikansk cent hver.',
    'nya': "Tsiku Lolemba, asayansi ochokera ku Stanford University School of Medicine analengeza zakupangidwa kwa chida chatsopano chomwe chitha kusiyanitsa ma cell ndi mtundu: kachipangizo kakang'ono kosindikizidwaka kakhoza kupangidwa pogwiritsa ntchito ma inkjet osindikizika apamwamba pafupifupi cent imodzi ya U.S iliyonse.",
    'oci': "Diluns, de scientifics de l'escòla de medecina de l'Universitat Stanford anoncièron l'invencion d'un novèl esplech de diagnostic que pòt triar las cellulas per tipe: una nièra minuscula imprimibla que pòt èsser manufacturada en utilizar d'estampadoiras de get de tencha per un còst possiblament a l'entorn d'un cent american caduna.",
    'ory': 'ସୋମବାର ଦିନ, ଷ୍ଟାନଫୋର୍ଡ ୟୁନିଭରସିଟି ସ୍କୁଲ୍ ଅଫ୍ ମେଡିସିନ୍ ର ବୈଜ୍ଞାନିକମାନେ ଏକ ନୂତନ ନିଦାନ ଉପକରଣର ଉଦ୍ଭାବନ ବିଷୟରେ ଘୋଷଣା କରିଛନ୍ତି ଯାହାର କୋଷଗୁଡିକ ପ୍ରକାର ଅନୁଯାୟୀ କ୍ରମବଦ୍ଧ କରାଯାଇପାରିବ: ଏକ କ୍ଷୁଦ୍ର ମୁଦ୍ରଣଯୋଗ୍ୟ ଚିପ୍ ଯାହାକି ସମ୍ଭବତଃ ପାଖାପାଖି 1 ପ୍ରତି ଆମେରିକୀୟ ସେଣ୍ଟରେ ଷ୍ଟାଣ୍ଡାର୍ଡ ଇଙ୍କଜେଟ୍ ମୁଦ୍ରକ ବ୍ୟବହାର କରି ଉତ୍ପାଦିତ ହୋଇପାରିବ।',
    'orm': 'Wixata, saayintistootni yuunivasitii Stanfordii kan kutaa barnoota fayyaa meeshaa haaraa yaaliif ta’uu kan seelii gosa gosaan adda addaa baasu beeksisan: waan xiqqaa maxxanfamuu danda’uu kan maxxansituu inkjeeti fayyadamuun maxxaansu kan tokkon tokko isaanii saantimaa U.S tokko ta’uu malaa.',
    'pus': 'د Stanford پوهنتون د طب د ځانګی ساینسپوهانو د دوه شنبی په ورځ د یوې داسې تشخیصی آلی د ایجاد اعلان وکړ چې کولای شي حجرات د نوعی له مخی وويشي ، یو کوچنی چپ چې چاپ کول یې په معیاری Inkject Printer سره په د یوه امریکایې ډالر د یو سنټ په بیه ممکن دي',
    'fas': 'روز دوشنبه، دانشمندان دانشکده پزشکی دانشگاه استنفورد از اختراع دستگاه تشخیصی جدیدی سخن گفتند که می\u200cتواند سلول\u200cها را براساس نوعشان مرتب\u200cسازی کند: نوعی تراشه قابل چاپ که بااستفاده از چاپگرهای استاندارد جوهرافشان قابل تولید است و هزینه هرکدام از آنها احتمالاً حدود یک سنت آمریکا در می\u200cآید.',
    'pol': 'W poniedziałek naukowcy ze Szkoły Medycznej Uniwersytetu Stanforda obwieścili wynalezienie nowego narzędzia diagnostycznego, sortującego komórki według rodzaju: jest to miniaturowy chip drukowany, który można wyprodukować za pomocą standardowych drukarek atramentowych przy koszcie szacowanym na jednego centa amerykańskiego za sztukę.',
    'por': 'Na segunda-feira, cientistas da Escola de Medicina da Universidade de Stanford anunciaram a invenção de uma nova ferramenta de diagnóstico que pode classificar células por tipo: um minúsculo chip imprimível que pode ser fabricado usando impressoras jato de tinta padrão por possivelmente cerca de um centavo de dólar cada.',
    'pan': 'ਸੋਮਵਾਰ ਨੂੰ, ਸਟੈਨਫੋਰਡ ਯੂਨੀਵਰਸਿਟੀ ਸਕੂਲ ਆਫ਼ ਮੈਡੀਸਿਨ ਦੇ ਵਿਗਿਆਨੀਆਂ ਨੇ ਇੱਕ ਨਵੇਂ ਡਾਇਗਨੋਸਟਿਕ ਟੂਲ ਦੇ ਅਵਿਸ਼ਕਾਰ ਬਾਰੇ ਘੋਸ਼ਣਾ ਕੀਤੀ ਜੋ ਕਿਸਮ ਮੁਤਾਬਕ ਸੈੱਲਾਂ ਦੀ ਛਾਂਟੀ ਕਰ ਸਕਦਾ ਹੈ: ਇੱਕ ਬਹੁਤ ਛੋਟੀ ਪ੍ਰਿੰਟ ਕਰਨਯੋਗ ਚਿੱਪ ਜਿਸਦਾ ਨਿਰਮਾਣ ਸਟੈਂਡਰਡ ਇੰਕਜੈੱਟ ਪ੍ਰਿੰਟਰਾਂ ਦੀ ਵਰਤੋਂ ਕਰਕੇ ਸੰਭਾਵੀ ਤੌਰ ‘ਤੇ ਲਗਭਗ ਇੱਕ ਯੂ.ਐਸ. ਸੈਂਟ ਦੇ ਖ਼ਰਚੇ ਵਿੱਚ ਕੀਤਾ ਜਾ ਸਕਦਾ ਹੈ।',
    'ron': 'Luni, oameni de știință de la Facultatea de Medicină a Universității Stanford au anunțat inventarea unui nou instrument de diagnosticare, care poate sorta celulele în funcție de tipul lor: un cip minuscul, printabil, care poate fi produs folosind imprimante obișnuite cu jet de cerneală, pentru aproximativ un cent american bucata.',
    'rus': 'В понедельник ученые из Медицинской школы Стэнфордского университета объявили об изобретении нового диагностического инструмента, который может сортировать клетки по их типу; это маленький чип, который можно напечатать, используя стандартный струйный принтер примерно за 1 цент США.',
    'srp': 'У понедељак су научници са медицинског факултета Универзитета Станфорд представили изум новог алата за дијагностику који може да поређа ћелије по типу: малени чип који се може штампати и који може да се произведе коришћењем стандардних инкџет штампача, и да кошта отприлике један амерички цент по комаду.',
    'sna': 'NeMuvhuru, mascientists vekuChikoro cheMishonga che yunivhesiti yekuStanford vakazivisa kugadzirwa kwe muchina mutsva wekunzvera uyo unogona kuronga macell nerudzi arwo: chi chip chinotsikiswa chinogadzirwa uchishandisa zvitsikiso zveinkhet awo anongangoita sendi rimwe chete remadhora.',
    'snd': 'سومر جي ڏينهن، اسٽينفورڊ يونيورسٽي اسڪول آف ميڊيسن جي سائنسدانن هڪ نئون تشخيصي ٽول ايجاد ڪرڻ جو اعلان ڪيو جيڪو جيو گهرڙن جي قسمن جي لحاظ سان ترتيب ڏيئي سگهي ٿو: هڪ ننڍي ڇپائيءِ جوڳي چپ جيڪو معياري انڪ جيٽ پرنٽرز جو استعمال ڪندي هر هڪ لاءِ لڳ ڀڳ هڪ U.S. سينٽ ۾ تيار ڪري سگهجي ٿي.',
    'slk': 'V pondelok vedci z lekárskej fakulty Stanfordskej univerzity informovali o objavení nového diagnostického prístroja, ktorý je schopný triediť bunky podľa druhu: malý čip, ktorý sa dá vytlačiť a je možné vyrobiť ho použitím štandardných atramentových tlačiarní, z ktorých každý môže mať hodnotu zhruba jeden cent.',
    'slv': 'Znanstveniki Medicinske fakultete Univerze v Stanfordu so v ponedeljek razglasili iznajdbo novega diagnostičnega orodja, ki je sposobno razvrščanja celic glede na vrsto: drobceno tiskano vezje, ki ga je mogoče izdelati z uporabo standardnih brizgalnih tiskalnikov za okoli en ameriški cent na kos.',
    'som': 'Maalinta Isniinta, saynis yahano ka socda Jamacada Stanford ee Iskuul Caafimadka waxay ku dhawaaqeen sameynta qalab daweyn cusub lagu habeyn karo unugyada : kuwa yar yar ee lagu soo saari karo muditan waxaana macquul ah in lagu soo saro Mareykan ka mid walba.',
    'ckb': 'لە دوو شەمەدا زانایانی سکوڵی پزیشكی لە زانکۆی ستانفۆرد رایانگەیاند کە داهێنانیان کردووە بۆ ئامرازێکی نوێی پشکنین کە دەتوانێت خانەکان پۆلێن بکات بەپێی جۆرەکان: ئامێرەکە پارچەیەکی ئەلیکترۆنیی بچوکە کە دەکرێت بە چاپکردن دروست بکرێت بە بەکارهێنانی چاپکەری مەرەکەب فڕێدەر بە تەنها یەك سەنتی ئەمریکی.',
    'spa': 'El lunes, los científicos de la facultad de medicina de la Universidad de Stanford anunciaron el invento de una nueva herramienta de diagnóstico que puede catalogar las células según su tipo: un pequeñísimo chip que se puede imprimir y fabricar con impresoras de inyección de uso corriente, por un posible costo de, aproximadamente, un centavo de dólar por cada uno.',
    'swh': 'Mnamo Jumatatu, wanasayansi kutoka Shule ya Tiba ya Chuo Kikuu cha Stanford walitangaza uvumbuzi wa kifaa kipya cha utambuzi ambacho kinaweza kupanga seli kwa aina: kidude kidogo kinachoweza kuchapwa, na ambacho kinaweza kutengenezwa kwa kutumia printa ya kawaida ya kupuliza rangi, yawezekana kwa takribani senti moja ya Marekani kwa kila moja.',
    'swe': 'I måndags meddelade forskare från Stanford University School of Medicine att man tagit fram ett nytt diagnostiskt verktyg som kan ordna celler efter typ: ett litet utskrivbart chip som kan tillverkas med vanliga bläckstråleskrivare för eventuellt ungefär en amerikansk cent vardera.',
    'tgk': 'Олимони мактаби тибби донишгоҳи Стенфорд рӯзи душанбе дар бораи ихтирои асбоби нави ташхисие, ки метавонад ҳуҷайраҳоро аз рӯи навъ мураттаб кунад, эълон карданд: чипи хурди чопшавандае, ки бо истифода аз принтерҳои фавракдами муқаррарӣ истеҳсол карда мешавад ва эҳтимолан ҳар як кадомаш тақрибан 1 сенти ИМА арзиш дорад.',
    'tam': 'திங்களன்று, ஸ்டான்போர்ட் யுனிவர்சிட்டி ஸ்கூல் ஆஃப் மெடிசின் விஞ்ஞானிகள் ஒரு புதிய நோயறிதல் கருவியின் கண்டுபிடிப்பை அறிவித்தனர், இது செல்களை வகைப்படி வரிசைப்படுத்தலாம்: ஒரு சிறிய அச்சிடக்கூடிய சில்லு, தரமான இன்க்ஜெட் அச்சுப்பொறிகளைப் பயன்படுத்தி ஒவ்வொன்றும் 1 யு.எஸ்.',
    'tel': 'సోమవారం, స్టాన్ఫోర్డ్ యూనివర్శిటీ స్కూల్ ఆఫ్ మెడిసిన్ శాస్త్రవేత్తలు కణాల రకాన్ని క్రమబద్ధీకరించగల కొత్త రోగనిర్ధారణ సాధనం యొక్క ఆవిష్కరణను ప్రకటించారు: ప్రామాణిక ఇంక్జెట్ ప్రింటర్లను ఉపయోగించి 1 యు.ఎస్.',
    'tha': 'เมื่อวันจันทร์ที่ผ่านมา นักวิทยาศาสตร์จากโรงเรียนการแพทย์แห่งมหาวิทยาลัยสแตนฟอร์ดได้ประกาศถึงการประดิษฐ์เครื่องมือวินิจฉัยใหม่ที่สามารถจัดเรียงเซลล์ตามประเภทได้ ซึ่งก็คือชิปขนาดเล็กซึ่งสามารถผลิตได้โดยใช้เครื่องพิมพ์อิงค์เจ็ตแบบมาตรฐานในราคาประมาณชิ้นละหนึ่งเซ็นต์ดอลลาร์สหรัฐ',
    'tur': 'Stanford Üniversitesi Tıp Fakültesi’nde görev alan bilim insanları, pazartesi günü hücreleri cinslerine göre sıraya koyabilen yeni bir teşhis aracının bulunduğunu duyurdu. Bu araç, standart mürekkep püskürtmeli yazıcılar kullanılarak her biri tahminen yaklaşık bir sente imal edilebilen küçük bir yazdırılabilir çipti.',
    'ukr': 'У понеділок, науковці зі Школи медицини Стенфордського університету оголосили про винайдення нового діагностичного інструменту, що може сортувати клітини за їх видами: це малесенький друкований чіп, який можна виготовити за допомогою стандартних променевих принтерів десь по одному центу США за штуку.',
    'umb': 'K’eteke lyatete lyosemana, ulongisi wosikola wesaku isangiwa vo Stanford vakonomwisa etungo lyocina cimwe cikwatisa okusanga oselula kwenda itava okuyitanga: okacina kaco katito calwa kwenda citava okuyupa l’okuyipanga l’onjanga yalwa kwenda yisukilañgo ovava l’osentimu Amerikanu imosi.',
    'urd': 'پیر کے روز، سٹینفورڈ اسکول آف میڈیسن کے سائنسدانوں نے ایک جدید تشخیصی آلہ دریافت کرنے کا اعلان کیا ہے جو خلیوں کو اس کی اقسام کے لحاظ سے ترتیب دے سکتا ہے: یہ ایک چھوٹی سی پرنٹیبل چپ ہے جو غالباً ایک امریکی سنٹ میں معیاری انک جیٹ پرنٹرز کا استعمال کر کے تیار کی جا سکتی ہے-',
    'uzb': "Dushanba kuni Stenford Universitetining Tibbiyot maktabi olimlari hujayralarni turlariga qarab saralay oladigan yangi tashxis vositasi ixtirosini e'lon qildi: har biri taxminan bir AQSH senti atrofida bo'lgan standart rangli printerlardan foydalangan holda ishlab chiqarish mumkin bo'lgan ingichka bosma chip.",
    'vie': 'Vào hôm thứ Hai, các nhà khoa học thuộc Trường Y Đại học Stanford đã công bố phát minh một dụng cụ chẩn đoán mới có thể phân loại tế bào: một con chíp nhỏ có thể sản xuất bằng máy in phun tiêu chuẩn với giá khoảng một xu Mỹ mỗi chiếc.',
    'cym': "Ar ddydd Llun, datganodd gwyddonwyr o Ysgol Feddygaeth Stanford eu bod wedi dyfeisio teclyn diagnostig newydd sy'n gallu didoli celloedd yn ôl math: sglodyn bach argraffadwy y gellir ei gynhyrchu gan ddefnyddio chwistrell-argraffwyr safonol am tua un sent yr U.D. yr un.",
    'wol': 'Ci Altine gi, xeltukat yi nekk Daara ju kawe ji di ajju ci wàllu Paj bu Stanford yëgle nañu seetub jumtukayu jagnostik bu bées bu mëna xaajale xeeti selul yi: pus bu ndaw bu nu imprme bu nu mëna defar jëfandikoo daa bu imprimant yi ngir lu toll ci benn dërëmu amerig bu nekk.',
    'xho': 'NgoMvulo, iinzululwazi ezivela eStandford University School of Medicine zazise ngesixhobo sokuxilonga esitsha esinoku sota iiseli ngohlobo: itship encinci eprintwayo enokwenziwa ngokusebenzisa iiprinta ze inki yaye zinokuxabisa malunga nesenti enye yase U.S..',
    'yor': 'Àwọn onímọ̀ ìjìnlẹ̀ sáyẹ̀nsì láti ilé ìkẹ́ẹ̀kọ́ gíga ti ìsègun Stanford lọ́jọ́ ajé ti kéde ìdásílẹ̀ irinṣẹ́ ìwádìí tuntun tí ó le tó nǹkan lẹ́sẹẹsẹ pẹ̀lú bí wọ́n bá se rí: irinṣẹ́ kéreké tí a lè tẹ̀ jáde pẹ̀lú lílo irinṣẹ́ ìtẹ̀wé ìgbàlódé pẹ̀lú owó ẹyọ fún ọ̀kọ̀ọ̀kan.',
    'zul': 'NgoMsombuluko, usosayensi wase-Stanford University School of Medicine umemezele ithuluzi Elisha elikwazi ukuhlukanisa amagqamuzane ngezinhlobo zawo: ucezwana oluncane olunyathelisekayo olungakhandwa ngokusebenzisa imishini evamile yokunyathelisa ngokusesilinganisweni ngesenti elilodwa laseMelika.',
}

translation_models/utils_llama.py
ZurichNLP/ContraDecode